[llvm] [LoadStoreVectorizer] Support vectorization of mixed-type contiguous accesses (PR #177908)
Anshil Gandhi via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 5 04:13:13 PDT 2026
https://github.com/gandhi56 updated https://github.com/llvm/llvm-project/pull/177908
>From c2c81143cbbbde08a7894ac471728ea5ff9832ed Mon Sep 17 00:00:00 2001
From: Anshil Gandhi <gandhi21299 at gmail.com>
Date: Sat, 20 Dec 2025 02:32:15 -0500
Subject: [PATCH] [LoadStoreVectorizer] Support vectorization of mixed-type
contiguous accesses
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Relaxes the LoadStoreVectorizer to support merging contiguous loads and
stores of different element types (e.g., merging an i32 and an i16 load).
Key changes:
- Remove `ElementSize` from the `EqClassKey`, allowing instructions with
different scalar types to be grouped together if they share the same
underlying object and address space.
- Calculate the vector element type using the Greatest Common Divisor (GCD)
of all scalar bitwidths in the chain.
- Add `splitChainByPointerWidth` to ensure pointer-typed elements whose bit-
width exceeds the GCD are partitioned into separate chains. This prevents
vectorizing pointers at a width smaller than their native size.
- Add `resizeAndBitcast` to handle mismatched type conversions during
vectorization (refactored to flatten → zext/trunc → reinterpret, preferring
bitcast for LLVM byte types bN so ptr/int crossings avoid unnecessary
ptrtoint/inttoptr where applicable).
- Update `vectorizeChain` to use shuffles, extracts, and bitcasts to
reconstruct original values from the vectorized result (for loads) or
prepare values for insertion (for stores).
- Update and add extensive tests covering mixed integer sizes, pointer
types, and floating-point mixes. Existing AMDGPU and X86 tests are
updated to reflect the more aggressive vectorization.
- Add merge_load_v2i32_i16 test
Resolves #97715
---
llvm/lib/IR/Value.cpp | 20 +-
.../Vectorize/LoadStoreVectorizer.cpp | 375 ++-
.../AMDGPU/32-bit-local-address-space.ll | 68 +-
.../AMDGPU/GlobalISel/atomicrmw_minmax.ll | 120 +-
.../AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll | 100 +-
.../AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll | 100 +-
.../AMDGPU/GlobalISel/extractelement.ll | 142 +-
.../irtranslator-call-return-values.ll | 28 +-
.../AMDGPU/GlobalISel/legalize-bitcast.mir | 8 +-
.../AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll | 170 +-
...gcn.raw.ptr.buffer.atomic.fadd-with-ret.ll | 2 +-
.../GlobalISel/llvm.amdgcn.set.inactive.ll | 181 +-
....struct.ptr.buffer.atomic.fadd-with-ret.ll | 2 +-
.../GlobalISel/llvm.amdgcn.trig.preop.ll | 17 +-
.../test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll | 926 +++---
.../test/CodeGen/AMDGPU/GlobalISel/udivrem.ll | 653 ++--
llvm/test/CodeGen/AMDGPU/abs_i32.ll | 9 +-
llvm/test/CodeGen/AMDGPU/add.v2i16.ll | 16 +-
llvm/test/CodeGen/AMDGPU/add_i128.ll | 14 +-
llvm/test/CodeGen/AMDGPU/addrspacecast.ll | 26 +-
.../AMDGPU/agpr-copy-no-free-registers.ll | 236 +-
llvm/test/CodeGen/AMDGPU/amd.endpgm.ll | 34 +-
.../CodeGen/AMDGPU/amdgcn.private-memory.ll | 8 +-
.../AMDGPU/amdgpu-codegenprepare-idiv.ll | 2578 +++++++--------
.../amdgpu-miscellaneous-uniform-intrinsic.ll | 4 +-
.../CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll | 4 +-
llvm/test/CodeGen/AMDGPU/and.ll | 178 +-
llvm/test/CodeGen/AMDGPU/andorbitset.ll | 90 +-
llvm/test/CodeGen/AMDGPU/andorn2.ll | 428 +--
llvm/test/CodeGen/AMDGPU/andorxorinvimm.ll | 90 +-
llvm/test/CodeGen/AMDGPU/anyext.ll | 51 +-
.../CodeGen/AMDGPU/asyncmark-gfx12plus.ll | 12 +-
.../CodeGen/AMDGPU/atomic_cmp_swap_local.ll | 119 +-
.../atomic_optimizations_global_pointer.ll | 14 +-
.../atomic_optimizations_local_pointer.ll | 423 ++-
.../CodeGen/AMDGPU/atomicrmw_usub_cond.ll | 122 +-
.../test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll | 472 ++-
.../AMDGPU/atomics-hw-remarks-gfx90a.ll | 16 +-
llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll | 4 +-
.../AMDGPU/av-split-dead-valno-crash.ll | 58 +-
llvm/test/CodeGen/AMDGPU/bfe-combine.ll | 60 +-
llvm/test/CodeGen/AMDGPU/bfe-patterns.ll | 84 +-
llvm/test/CodeGen/AMDGPU/bfi_int.ll | 126 +-
llvm/test/CodeGen/AMDGPU/bfi_nested.ll | 17 +-
llvm/test/CodeGen/AMDGPU/bfm.ll | 29 +-
llvm/test/CodeGen/AMDGPU/bitreverse.ll | 116 +-
...der-no-live-segment-at-def-implicit-def.ll | 27 +-
.../branch-folding-implicit-def-subreg.ll | 578 ++--
.../AMDGPU/branch-relaxation-gfx1250.ll | 274 +-
llvm/test/CodeGen/AMDGPU/branch-relaxation.ll | 552 ++--
.../AMDGPU/buffer-intrinsics-mmo-offsets.ll | 359 ++-
llvm/test/CodeGen/AMDGPU/build_vector.ll | 68 +-
.../test/CodeGen/AMDGPU/carryout-selection.ll | 74 +-
llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll | 118 +-
.../CodeGen/AMDGPU/combine-cond-add-sub.ll | 185 +-
.../CodeGen/AMDGPU/combine-reg-or-const.ll | 28 +-
.../AMDGPU/constant-fold-mi-operands.ll | 10 +-
llvm/test/CodeGen/AMDGPU/ctlz.ll | 30 +-
llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll | 69 +-
llvm/test/CodeGen/AMDGPU/ctpop.ll | 51 +-
llvm/test/CodeGen/AMDGPU/ctpop16.ll | 34 +-
llvm/test/CodeGen/AMDGPU/ctpop64.ll | 90 +-
llvm/test/CodeGen/AMDGPU/cttz.ll | 26 +-
llvm/test/CodeGen/AMDGPU/cttz_zero_poison.ll | 48 +-
llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll | 9 -
llvm/test/CodeGen/AMDGPU/cvt_flr_i32_f32.ll | 102 +-
llvm/test/CodeGen/AMDGPU/cvt_rpi_i32_f32.ll | 85 +-
...dagcomb-extract-vec-elt-different-sizes.ll | 36 +-
llvm/test/CodeGen/AMDGPU/dagcombine-select.ll | 212 +-
.../test/CodeGen/AMDGPU/dbg-info-inline-at.ll | 14 +-
.../AMDGPU/divergence-driven-buildvector.ll | 77 +-
.../AMDGPU/divergence-driven-sext-inreg.ll | 56 +-
.../AMDGPU/divergence-driven-trunc-to-i1.ll | 88 +-
.../AMDGPU/ds-combine-with-dependence.ll | 1 +
llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll | 10 +-
llvm/test/CodeGen/AMDGPU/ds_read2.ll | 169 +-
llvm/test/CodeGen/AMDGPU/ds_write2.ll | 33 +-
.../CodeGen/AMDGPU/extract_vector_dynelt.ll | 860 +++--
.../CodeGen/AMDGPU/extract_vector_elt-f16.ll | 41 +-
.../CodeGen/AMDGPU/extract_vector_elt-i16.ll | 29 +-
.../CodeGen/AMDGPU/extract_vector_elt-i64.ll | 2 -
.../CodeGen/AMDGPU/extract_vector_elt-i8.ll | 24 +-
llvm/test/CodeGen/AMDGPU/fabs.bf16.ll | 195 +-
llvm/test/CodeGen/AMDGPU/fabs.f16.ll | 150 +-
llvm/test/CodeGen/AMDGPU/fabs.ll | 97 +-
llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll | 56 +-
llvm/test/CodeGen/AMDGPU/fcanonicalize.ll | 92 +-
llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll | 419 ++-
llvm/test/CodeGen/AMDGPU/fdiv.ll | 669 ++--
.../CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll | 6 +-
.../test/CodeGen/AMDGPU/flat-address-space.ll | 11 +-
llvm/test/CodeGen/AMDGPU/flat-scratch-init.ll | 13 +-
llvm/test/CodeGen/AMDGPU/flat_atomics.ll | 2097 ++++++-------
.../CodeGen/AMDGPU/flat_atomics_i32_system.ll | 39 +-
llvm/test/CodeGen/AMDGPU/fma.f16.gfx11plus.ll | 26 +-
llvm/test/CodeGen/AMDGPU/fmaxnum.ll | 167 +-
llvm/test/CodeGen/AMDGPU/fmin_legacy.ll | 10 +-
llvm/test/CodeGen/AMDGPU/fminnum.ll | 167 +-
.../AMDGPU/fmul-2-combine-multi-use.ll | 395 +--
llvm/test/CodeGen/AMDGPU/fnearbyint.ll | 79 +-
llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll | 74 +-
llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll | 510 ++-
llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll | 326 +-
llvm/test/CodeGen/AMDGPU/fneg-fabs.ll | 63 +-
llvm/test/CodeGen/AMDGPU/fneg.bf16.ll | 170 +-
llvm/test/CodeGen/AMDGPU/fneg.f16.ll | 204 +-
llvm/test/CodeGen/AMDGPU/fneg.ll | 270 +-
llvm/test/CodeGen/AMDGPU/fp-classify.ll | 396 ++-
.../CodeGen/AMDGPU/fp64-atomics-gfx90a.ll | 672 ++--
.../AMDGPU/fp64-min-max-buffer-atomics.ll | 101 +-
.../AMDGPU/fp64-min-max-buffer-ptr-atomics.ll | 101 +-
llvm/test/CodeGen/AMDGPU/fp_to_sint.ll | 307 +-
llvm/test/CodeGen/AMDGPU/fp_to_uint.ll | 234 +-
llvm/test/CodeGen/AMDGPU/fpext.f16.ll | 22 +-
llvm/test/CodeGen/AMDGPU/fptosi.f16.ll | 40 +-
llvm/test/CodeGen/AMDGPU/fptoui.f16.ll | 35 +-
llvm/test/CodeGen/AMDGPU/fptoui_uitofp.ll | 81 +-
llvm/test/CodeGen/AMDGPU/fshl.ll | 433 ++-
llvm/test/CodeGen/AMDGPU/fshr.ll | 513 ++-
llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll | 93 +-
.../AMDGPU/gfx-callable-argument-types.ll | 48 +-
llvm/test/CodeGen/AMDGPU/global_atomics.ll | 2756 ++++++++---------
.../AMDGPU/global_atomics_i32_system.ll | 53 +-
llvm/test/CodeGen/AMDGPU/half.ll | 485 +--
llvm/test/CodeGen/AMDGPU/idiv-licm.ll | 498 ++-
llvm/test/CodeGen/AMDGPU/iglp-no-clobber.ll | 27 +-
llvm/test/CodeGen/AMDGPU/imm.ll | 681 ++--
llvm/test/CodeGen/AMDGPU/imm16.ll | 546 ++--
llvm/test/CodeGen/AMDGPU/immv216.ll | 214 +-
.../CodeGen/AMDGPU/indirect-addressing-si.ll | 1026 +++---
llvm/test/CodeGen/AMDGPU/inlineasm-sgmask.ll | 1 +
.../CodeGen/AMDGPU/insert-waitcnts-merge.ll | 33 +-
.../CodeGen/AMDGPU/insert_vector_dynelt.ll | 140 +-
llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll | 192 +-
.../insert_waitcnt_for_precise_memory.ll | 137 +-
llvm/test/CodeGen/AMDGPU/kernel-args.ll | 430 +--
.../CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll | 120 +-
.../CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll | 52 +-
.../CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll | 52 +-
.../CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll | 84 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.ll | 60 +-
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.i16.ll | 74 +-
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.u16.ll | 74 +-
.../AMDGPU/llvm.amdgcn.cvt.pknorm.i16.ll | 18 +-
.../AMDGPU/llvm.amdgcn.cvt.pknorm.u16.ll | 18 +-
.../CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll | 82 +-
.../CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll | 60 +-
.../CodeGen/AMDGPU/llvm.amdgcn.ds.swizzle.ll | 28 +-
.../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll | 240 +-
.../CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll | 2600 ++++++++++------
.../CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.ll | 100 +-
.../CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll | 160 +-
.../CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll | 1712 ++++++----
.../CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll | 1012 +++---
....amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll | 32 +-
.../CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll | 8 +-
.../CodeGen/AMDGPU/llvm.amdgcn.permlane64.ll | 4 +-
.../AMDGPU/llvm.amdgcn.permlane64.ptr.ll | 12 +-
...llvm.amdgcn.raw.buffer.store.format.d16.ll | 16 +-
....amdgcn.raw.ptr.buffer.store.format.d16.ll | 16 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll | 587 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll | 354 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll | 479 ++-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll | 18 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll | 18 +-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll | 479 ++-
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll | 354 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll | 354 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll | 354 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll | 631 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll | 354 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll | 354 +--
.../CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll | 585 ++--
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll | 161 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll | 16 +-
.../CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll | 38 +-
.../AMDGPU/llvm.amdgcn.set.inactive.ll | 181 +-
...m.amdgcn.struct.buffer.store.format.d16.ll | 16 +-
...dgcn.struct.ptr.buffer.store.format.d16.ll | 16 +-
...lvm.amdgcn.struct.ptr.tbuffer.store.d16.ll | 50 +-
.../llvm.amdgcn.struct.tbuffer.store.d16.ll | 50 +-
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ubfe.ll | 126 +-
.../CodeGen/AMDGPU/llvm.amdgcn.writelane.ll | 367 +--
llvm/test/CodeGen/AMDGPU/llvm.exp.ll | 179 +-
llvm/test/CodeGen/AMDGPU/llvm.exp10.ll | 179 +-
llvm/test/CodeGen/AMDGPU/llvm.exp2.ll | 108 +-
.../CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll | 41 +-
.../CodeGen/AMDGPU/llvm.is.fpclass.f16.ll | 41 +-
llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll | 33 +-
llvm/test/CodeGen/AMDGPU/llvm.log.ll | 369 ++-
llvm/test/CodeGen/AMDGPU/llvm.log10.ll | 369 ++-
llvm/test/CodeGen/AMDGPU/llvm.log2.ll | 86 +-
llvm/test/CodeGen/AMDGPU/llvm.round.ll | 293 +-
llvm/test/CodeGen/AMDGPU/load-select-ptr.ll | 12 +-
.../CodeGen/AMDGPU/local-atomicrmw-fadd.ll | 783 +++--
llvm/test/CodeGen/AMDGPU/local-atomics.ll | 22 +-
llvm/test/CodeGen/AMDGPU/local-atomics64.ll | 21 +-
.../AMDGPU/long-branch-reserve-register.ll | 50 +-
.../CodeGen/AMDGPU/lower-lds-struct-aa.ll | 17 +-
.../AMDGPU/lower-lds-with-alias-scope.ll | 7 +-
.../CodeGen/AMDGPU/lower-lds-with-noalias.ll | 7 +-
llvm/test/CodeGen/AMDGPU/lshl64-to-32.ll | 74 +-
llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll | 41 +-
.../test/CodeGen/AMDGPU/machine-sink-fence.ll | 6 +-
...ne-sink-temporal-divergence-swdev407790.ll | 33 +-
llvm/test/CodeGen/AMDGPU/mad_64_32.ll | 6 +-
llvm/test/CodeGen/AMDGPU/mad_int24.ll | 28 +-
llvm/test/CodeGen/AMDGPU/mad_uint24.ll | 268 +-
llvm/test/CodeGen/AMDGPU/madak.ll | 36 +-
llvm/test/CodeGen/AMDGPU/max.ll | 116 +-
llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll | 188 +-
...-legalizer-single-wave-workgroup-memops.ll | 198 +-
llvm/test/CodeGen/AMDGPU/memory_clause.ll | 6 +-
.../AMDGPU/memset-param-combinations.ll | 135 +-
llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll | 52 +-
llvm/test/CodeGen/AMDGPU/mfma-loop.ll | 218 +-
llvm/test/CodeGen/AMDGPU/min.ll | 387 ++-
.../CodeGen/AMDGPU/misched-remat-revert.ll | 576 ++++
llvm/test/CodeGen/AMDGPU/mul.ll | 412 ++-
llvm/test/CodeGen/AMDGPU/mul_int24.ll | 194 +-
llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll | 192 +-
.../test/CodeGen/AMDGPU/no-shrink-extloads.ll | 24 +-
llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll | 24 +-
llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll | 82 +-
llvm/test/CodeGen/AMDGPU/or.ll | 76 +-
llvm/test/CodeGen/AMDGPU/packed-fp32.ll | 151 +-
llvm/test/CodeGen/AMDGPU/permlane-op-sel.ll | 4 +-
llvm/test/CodeGen/AMDGPU/permute.ll | 39 +-
llvm/test/CodeGen/AMDGPU/permute_i8.ll | 34 +-
llvm/test/CodeGen/AMDGPU/pr155452.ll | 49 +-
.../CodeGen/AMDGPU/private-memory-atomics.ll | 34 +-
...-alloca-vector-dynamic-idx-bitcasts-llc.ll | 36 +-
.../AMDGPU/ptradd-sdag-optimizations.ll | 9 +-
llvm/test/CodeGen/AMDGPU/rcp-pattern.ll | 84 +-
.../AMDGPU/reassoc-mul-add-1-to-mad.ll | 179 +-
.../AMDGPU/reg-coalescer-sched-crash.ll | 22 +-
llvm/test/CodeGen/AMDGPU/returnaddress.ll | 3 +-
llvm/test/CodeGen/AMDGPU/rotl.ll | 116 +-
llvm/test/CodeGen/AMDGPU/rotr.ll | 106 +-
llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll | 30 +-
llvm/test/CodeGen/AMDGPU/rsq.f32.ll | 10 +-
llvm/test/CodeGen/AMDGPU/s_addk_i32.ll | 8 +-
llvm/test/CodeGen/AMDGPU/s_mulk_i32.ll | 120 +-
llvm/test/CodeGen/AMDGPU/sad.ll | 219 +-
llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll | 32 +-
llvm/test/CodeGen/AMDGPU/sdwa-commute.ll | 2 +-
.../AMDGPU/sdwa-peephole-instr-combine-sel.ll | 4 +-
llvm/test/CodeGen/AMDGPU/select-vectors.ll | 2 -
llvm/test/CodeGen/AMDGPU/setcc-opt.ll | 13 +-
.../AMDGPU/sext-divergence-driven-isel.ll | 98 +-
llvm/test/CodeGen/AMDGPU/sext-in-reg.ll | 18 +-
llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll | 23 +-
.../CodeGen/AMDGPU/sgpr-copy-local-cse.ll | 8 +-
llvm/test/CodeGen/AMDGPU/shl.ll | 30 +-
llvm/test/CodeGen/AMDGPU/shl.v2i16.ll | 66 +-
llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll | 32 +-
.../AMDGPU/si-triv-disjoint-mem-access.ll | 90 +-
.../si-unify-exit-multiple-unreachables.ll | 85 +-
llvm/test/CodeGen/AMDGPU/sign_extend.ll | 284 +-
.../siloadstoreopt-misaligned-regsequence.ll | 10 +-
.../AMDGPU/simulated-trap-pseudo-expand.ll | 15 +-
llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll | 342 +-
llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll | 281 +-
.../AMDGPU/splitkit-getsubrangeformask.ll | 62 +-
llvm/test/CodeGen/AMDGPU/sra.ll | 38 +-
llvm/test/CodeGen/AMDGPU/store-to-constant.ll | 30 +-
llvm/test/CodeGen/AMDGPU/structurize-hoist.ll | 29 +-
llvm/test/CodeGen/AMDGPU/sub.ll | 42 +-
llvm/test/CodeGen/AMDGPU/sub.v2i16.ll | 41 +-
.../AMDGPU/subreg-coalescer-undef-use.ll | 10 +-
llvm/test/CodeGen/AMDGPU/trunc-store-i1.ll | 8 +-
llvm/test/CodeGen/AMDGPU/trunc.ll | 44 +-
.../AMDGPU/tuple-allocation-failure.ll | 106 +-
llvm/test/CodeGen/AMDGPU/uaddo.ll | 119 +-
llvm/test/CodeGen/AMDGPU/udiv.ll | 90 +-
llvm/test/CodeGen/AMDGPU/udivrem.ll | 126 +-
llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll | 344 +-
llvm/test/CodeGen/AMDGPU/uniform-cfg.ll | 162 +-
.../CodeGen/AMDGPU/use-sgpr-multiple-times.ll | 30 +-
llvm/test/CodeGen/AMDGPU/usubo.ll | 119 +-
llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll | 124 +-
llvm/test/CodeGen/AMDGPU/v_cndmask.ll | 284 +-
.../AMDGPU/vectorize-buffer-fat-pointer.ll | 4 +-
.../waitcnt-loop-ds-prefetch-flushed.ll | 20 +-
.../waitcnt-loop-ds-prefetch-pattern.ll | 40 +-
llvm/test/CodeGen/AMDGPU/wave32.ll | 76 +-
llvm/test/CodeGen/AMDGPU/xor.ll | 39 +-
.../AMDGPU/zext-divergence-driven-isel.ll | 64 +-
.../machine-function-info-long-branch-reg.ll | 2 +-
llvm/test/CodeGen/NVPTX/dag-cse.ll | 6 +-
llvm/test/CodeGen/NVPTX/variadics-backend.ll | 10 +-
llvm/test/CodeGen/X86/loadStore_vectorizer.ll | 3 +-
.../LoadStoreVectorizer/AMDGPU/aa-metadata.ll | 14 +-
.../AMDGPU/adjust-alloca-alignment.ll | 100 +-
.../AMDGPU/complex-index.ll | 46 +-
.../AMDGPU/dont-adjust-globalobj-alignment.ll | 9 +-
.../AMDGPU/extended-index.ll | 118 +-
.../AMDGPU/fence-noalias-metadata.ll | 16 +-
.../LoadStoreVectorizer/AMDGPU/gep-bitcast.ll | 124 +-
.../AMDGPU/insertion-point.ll | 79 +-
.../AMDGPU/invariant-load.ll | 18 +-
.../AMDGPU/merge-stores-private.ll | 352 ++-
.../AMDGPU/merge-stores.ll | 506 +--
.../AMDGPU/merge-vectors-complex.ll | 255 +-
.../AMDGPU/merge-vectors.ll | 69 +-
.../AMDGPU/missing-alignment.ll | 10 +-
.../AMDGPU/mixed-ptr-and-narrow-int.ll | 88 +
.../AMDGPU/multiple_tails.ll | 34 +-
.../AMDGPU/no-implicit-float.ll | 17 +-
.../LoadStoreVectorizer/AMDGPU/opaque_ptr.ll | 8 +-
.../LoadStoreVectorizer/AMDGPU/optnone.ll | 18 +-
.../AMDGPU/pointer-elements.ll | 307 +-
.../AMDGPU/selects-inseltpoison.ll | 71 +-
.../LoadStoreVectorizer/AMDGPU/selects.ll | 77 +-
.../AMDGPU/store_with_aliasing_load.ll | 41 +-
.../AMDGPU/unaligned-buffer.ll | 42 +-
.../AMDGPU/vect-ptr-ptr-size-mismatch.ll | 117 +-
.../AMDGPU/vectorize-redund-loads.ll | 72 +-
.../AMDGPU/weird-type-accesses.ll | 177 +-
.../LoadStoreVectorizer/NVPTX/4x2xhalf.ll | 64 +-
.../LoadStoreVectorizer/NVPTX/extend-chain.ll | 68 +-
.../NVPTX/gap-fill-cleanup.ll | 14 +-
.../NVPTX/gap-fill-invariant.ll | 42 +-
.../NVPTX/gap-fill-vectors.ll | 72 +-
.../NVPTX/gap-fill-with-redundant-elements.ll | 55 +-
.../LoadStoreVectorizer/NVPTX/gap-fill.ll | 142 +-
.../NVPTX/load-store-256-bit.ll | 993 ++++--
.../NVPTX/many_loads_stores.ll | 7 +-
.../LoadStoreVectorizer/NVPTX/masked-store.ll | 269 +-
.../NVPTX/merge-across-side-effects.ll | 168 +-
.../NVPTX/non-instr-bitcast.ll | 14 +-
.../NVPTX/overlapping_chains.ll | 15 +-
.../NVPTX/propagate-invariance-metadata.ll | 8 +-
.../LoadStoreVectorizer/NVPTX/vectorize_i1.ll | 24 +-
.../NVPTX/vectorize_i16.ll | 19 +-
.../NVPTX/vectorize_i24.ll | 16 +-
.../LoadStoreVectorizer/NVPTX/vectorize_i8.ll | 469 ++-
.../NVPTX/vectorize_vectors.ll | 15 +-
.../codegenprepare-produced-address-math.ll | 55 +-
.../X86/compare-scev-by-complexity.ll | 57 +-
.../LoadStoreVectorizer/X86/correct-order.ll | 18 +-
.../X86/load-width-inseltpoison.ll | 120 +-
.../LoadStoreVectorizer/X86/load-width.ll | 120 +-
.../X86/massive_indirection.ll | 37 +-
.../LoadStoreVectorizer/X86/merge-tbaa.ll | 28 +-
.../LoadStoreVectorizer/X86/opaque-ptr.ll | 10 +-
.../X86/or-disjoint-nested-add.ll | 28 +-
.../X86/or-disjoint-zext.ll | 22 +-
.../X86/preserve-order32.ll | 17 +-
.../X86/preserve-order64.ll | 55 +-
.../X86/scev-range-zext.ll | 8 +-
.../X86/subchain-interleaved.ll | 132 +-
.../LoadStoreVectorizer/X86/vector-scalar.ll | 12 +-
.../vectorize-i8-nested-add-inseltpoison.ll | 20 +-
.../X86/vectorize-i8-nested-add.ll | 172 +-
.../X86/vectorize-redund-loads.ll | 16 +-
.../batch-aa-compile-time.ll | 2567 ++++++++++++++-
.../LoadStoreVectorizer/int_sideeffect.ll | 56 +-
358 files changed, 32690 insertions(+), 28863 deletions(-)
create mode 100644 llvm/test/CodeGen/AMDGPU/misched-remat-revert.ll
create mode 100644 llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/mixed-ptr-and-narrow-int.ll
diff --git a/llvm/lib/IR/Value.cpp b/llvm/lib/IR/Value.cpp
index 850acf2b7a33b..6c6d24b364185 100644
--- a/llvm/lib/IR/Value.cpp
+++ b/llvm/lib/IR/Value.cpp
@@ -783,17 +783,25 @@ const Value *Value::stripAndAccumulateConstantOffsets(
}
}
V = GEP->getPointerOperand();
- } else if (Operator::getOpcode(V) == Instruction::BitCast ||
- Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
+ } else if (Operator::getOpcode(V) == Instruction::BitCast) {
+ // Match stripPointerCastsAndOffsets: only look through ptr->ptr (or
+ // ptr-vector) bitcasts. Do not follow bitcast from byte/int/etc. to ptr
+ // (e.g. lossless bN -> ptr reconstitution) — the operand is not a
+ // pointer and pointer offset stripping cannot continue.
+ const Value *Src = cast<Operator>(V)->getOperand(0);
+ if (!Src->getType()->isPtrOrPtrVectorTy())
+ return V;
+ V = Src;
+ } else if (Operator::getOpcode(V) == Instruction::AddrSpaceCast) {
V = cast<Operator>(V)->getOperand(0);
} else if (auto *GA = dyn_cast<GlobalAlias>(V)) {
if (!GA->isInterposable())
V = GA->getAliasee();
} else if (const auto *Call = dyn_cast<CallBase>(V)) {
- if (const Value *RV = Call->getReturnedArgOperand())
- V = RV;
- if (AllowInvariantGroup && Call->isLaunderOrStripInvariantGroup())
- V = Call->getArgOperand(0);
+ if (const Value *RV = Call->getReturnedArgOperand())
+ V = RV;
+ if (AllowInvariantGroup && Call->isLaunderOrStripInvariantGroup())
+ V = Call->getArgOperand(0);
} else if (auto *Int2Ptr = dyn_cast<Operator>(V)) {
// Try to accumulate across (inttoptr (add (ptrtoint p), off)).
if (!AllowNonInbounds || !LookThroughIntToPtr || !Int2Ptr ||
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index 3f67e7ba9872b..9a764230d66d1 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -126,21 +126,16 @@ namespace {
// Equivalence class key, the initial tuple by which we group loads/stores.
// Loads/stores with different EqClassKeys are never merged.
-//
-// (We could in theory remove element-size from the this tuple. We'd just need
-// to fix up the vector packing/unpacking code.)
using EqClassKey =
std::tuple<const Value * /* result of getUnderlyingObject() */,
unsigned /* AddrSpace */,
- unsigned /* Load/Store element size bits */,
char /* IsLoad; char b/c bool can't be a DenseMap key */
>;
[[maybe_unused]] llvm::raw_ostream &operator<<(llvm::raw_ostream &OS,
const EqClassKey &K) {
- const auto &[UnderlyingObject, AddrSpace, ElementSize, IsLoad] = K;
+ const auto &[UnderlyingObject, AddrSpace, IsLoad] = K;
return OS << (IsLoad ? "load" : "store") << " of " << *UnderlyingObject
- << " of element size " << ElementSize << " bits in addrspace "
- << AddrSpace;
+ << " in addrspace " << AddrSpace;
}
// A Chain is a set of instructions such that:
@@ -154,13 +149,58 @@ using EqClassKey =
// - BB order, sorted by Instr->comesBefore.
// - Offset order, sorted by OffsetFromLeader.
// This pass switches back and forth between these orders.
+class Vectorizer;
+
struct ChainElem {
Instruction *Inst;
APInt OffsetFromLeader;
ChainElem(Instruction *Inst, APInt OffsetFromLeader)
: Inst(std::move(Inst)), OffsetFromLeader(std::move(OffsetFromLeader)) {}
};
-using Chain = SmallVector<ChainElem, 1>;
+struct Chain : SmallVector<ChainElem, 1> {
+ friend class Vectorizer;
+
+private:
+ /// Cached result of \p computeAndCacheElemTy.
+ Type *ElemTy = nullptr;
+
+public:
+ using SmallVector::SmallVector;
+
+ /// Return the GCD of the element sizes.
+ unsigned int computeGCDSize(const DataLayout &DL) const {
+ unsigned int GCDSize = 0;
+ for (const ChainElem &E : *this) {
+ Type *Ty = getLoadStoreType(E.Inst)->getScalarType();
+ unsigned Sz = DL.getTypeSizeInBits(Ty);
+ GCDSize = GCDSize == 0 ? Sz : std::gcd(GCDSize, Sz);
+ }
+ return GCDSize;
+ }
+
+ /// Gets the element type of the vector that the chain will load or store.
+ ///
+ /// The lane width is the GCD of scalar sizes in the chain. Scalars are
+ /// normalized to \p b<GCD> (LLVM byte types) so loads/stores use untyped byte
+ /// lanes and \p resizeAndBitcast restores int/float/pointer types without
+ /// ptrtoint/inttoptr for pointers. For sub-byte GCD widths (\p GCD < 8),
+ /// uses \p i<GCD> instead of \p b<GCD> because narrow byte vectors are less
+ /// common in codegen than \p i1/\p i4 chains.
+ ///
+ /// \returns The chosen element type for the vectorized load or store.
+ Type *computeAndCacheElemTy(LLVMContext &Ctx, const DataLayout &DL) {
+ assert(!empty());
+ if (ElemTy)
+ return ElemTy;
+
+ unsigned GCDSize = computeGCDSize(DL);
+ if (GCDSize < 8)
+ return ElemTy = Type::getIntNTy(Ctx, GCDSize);
+ LLVM_DEBUG(dbgs() << "LSV: using normalized byte element type b" << GCDSize
+ << "\n");
+ return ElemTy = Type::getByteNTy(Ctx, GCDSize);
+ }
+};
void sortChainInBBOrder(Chain &C) {
sort(C, [](auto &A, auto &B) { return A.Inst->comesBefore(B.Inst); });
@@ -298,6 +338,18 @@ class Vectorizer {
/// Discards any length-1 subchains.
std::vector<Chain> splitChainByAlignment(Chain &C);
+ /// Inserts casts to convert \p V to \p NewTy, preserving the memory bit
+ /// pattern. Normalized lane values (\p bN or \p iN) are flattened with
+ /// \p bitcast when widths differ, resized with \p zext/\p trunc on integers,
+ /// then reinterpreted—preferring \p bitcast (LLVM's encoding of byte/type
+ /// reinterpretation) whenever a byte type is involved so pointers avoid
+ /// unnecessary \p ptrtoint/\p inttoptr.
+ ///
+ /// \param V The value to be cast.
+ /// \param NewTy The target type for the value.
+ /// \returns The resulting casted Value.
+ Value *resizeAndBitcast(Value *V, Type *NewTy);
+
/// Converts the instrs in the chain into a single vectorized load or store.
/// Adds the old scalar loads/stores to ToErase.
bool vectorizeChain(Chain &C);
@@ -313,11 +365,6 @@ class Vectorizer {
Instruction *ContextInst,
unsigned Depth);
- /// Gets the element type of the vector that the chain will load or store.
- /// This is nontrivial because the chain may contain elements of different
- /// types; e.g. it's legal to have a chain that contains both i32 and float.
- Type *getChainElemTy(const Chain &C);
-
/// Determines whether ChainElem can be moved up (if IsLoad) or down (if
/// !IsLoad) to ChainBegin -- i.e. there are no intervening may-alias
/// instructions.
@@ -667,7 +714,7 @@ std::vector<Chain> Vectorizer::splitChainByContiguity(Chain &C) {
// and max vector size, as splitChainByAlignment will ensure the final vector
// shape passes the legalization check.
unsigned AS = getLoadStoreAddressSpace(C[0].Inst);
- Type *ElementType = getLoadStoreType(C[0].Inst)->getScalarType();
+ Type *ElementType = C.computeAndCacheElemTy(F.getContext(), DL);
unsigned MaxVecRegBits = TTI.getLoadStoreVecRegBitWidth(AS);
Align OptimisticAlign = Align(MaxVecRegBits / 8);
unsigned int MaxVectorNumElems =
@@ -712,7 +759,6 @@ std::vector<Chain> Vectorizer::splitChainByContiguity(Chain &C) {
std::vector<Chain> Ret;
Ret.push_back({C.front()});
- unsigned ChainElemTyBits = DL.getTypeSizeInBits(getChainElemTy(C));
ChainElem &Prev = C[0];
for (auto It = std::next(C.begin()), End = C.end(); It != End; ++It) {
auto &CurChain = Ret.back();
@@ -721,19 +767,24 @@ std::vector<Chain> Vectorizer::splitChainByContiguity(Chain &C) {
APInt(ASPtrBits, DL.getTypeStoreSize(getLoadStoreType(Prev.Inst)));
APInt PrevReadEnd = Prev.OffsetFromLeader + PrevSzBytes;
unsigned SzBytes = DL.getTypeStoreSize(getLoadStoreType(It->Inst));
-
- // Add this instruction to the end of the current chain, or start a new one.
- assert(
- 8 * SzBytes % ChainElemTyBits == 0 &&
- "Every chain-element size must be a multiple of the element size after "
- "vectorization.");
APInt ReadEnd = It->OffsetFromLeader + SzBytes;
- // Allow redundancy: partial or full overlap counts as contiguous.
+
+ // Allow redundancy: partial or full overlap counts as contiguous,
+ // provided the overlap aligns to the pairwise minimum scalar size of the
+ // two elements. This is sufficient because the chain's GCD element size
+ // always divides the pairwise minimum (all scalar sizes are powers of 2),
+ // and this property is preserved regardless of how later passes split the
+ // chain.
bool AreContiguous = false;
if (It->OffsetFromLeader.sle(PrevReadEnd)) {
- // Check overlap is a multiple of the element size after vectorization.
+ unsigned PrevScalarBits =
+ DL.getTypeSizeInBits(getLoadStoreType(Prev.Inst)->getScalarType());
+ unsigned ItScalarBits =
+ DL.getTypeSizeInBits(getLoadStoreType(It->Inst)->getScalarType());
+ assert(isPowerOf2_32(PrevScalarBits) && isPowerOf2_32(ItScalarBits) &&
+ "Pairwise overlap check assumes power-of-2 scalar sizes.");
uint64_t Overlap = (PrevReadEnd - It->OffsetFromLeader).getZExtValue();
- if (8 * Overlap % ChainElemTyBits == 0)
+ if (8 * Overlap % std::min(PrevScalarBits, ItScalarBits) == 0)
AreContiguous = true;
}
@@ -791,33 +842,6 @@ std::vector<Chain> Vectorizer::splitChainByContiguity(Chain &C) {
return Ret;
}
-Type *Vectorizer::getChainElemTy(const Chain &C) {
- assert(!C.empty());
- // The rules are:
- // - If there are any pointer types in the chain, use an integer type.
- // - Prefer an integer type if it appears in the chain.
- // - Otherwise, use the first type in the chain.
- //
- // The rule about pointer types is a simplification when we merge e.g. a load
- // of a ptr and a double. There's no direct conversion from a ptr to a
- // double; it requires a ptrtoint followed by a bitcast.
- //
- // It's unclear to me if the other rules have any practical effect, but we do
- // it to match this pass's previous behavior.
- if (any_of(C, [](const ChainElem &E) {
- return getLoadStoreType(E.Inst)->getScalarType()->isPointerTy();
- })) {
- return Type::getIntNTy(
- F.getContext(),
- DL.getTypeSizeInBits(getLoadStoreType(C[0].Inst)->getScalarType()));
- }
-
- for (const ChainElem &E : C)
- if (Type *T = getLoadStoreType(E.Inst)->getScalarType(); T->isIntegerTy())
- return T;
- return getLoadStoreType(C[0].Inst)->getScalarType();
-}
-
std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
// We use a simple greedy algorithm.
// - Given a chain of length N, find all prefixes that
@@ -895,7 +919,7 @@ std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
dbgs() << "LSV: splitChainByAlignment considering candidate chain ["
<< *C[CBegin].Inst << " ... " << *C[CEnd].Inst << "]\n");
- Type *VecElemTy = getChainElemTy(C);
+ Type *VecElemTy = C.computeAndCacheElemTy(F.getContext(), DL);
// Note, VecElemTy is a power of 2, but might be less than one byte. For
// example, we can vectorize 2 x <2 x i4> to <4 x i4>, and in this case
// VecElemTy would be i4.
@@ -1075,6 +1099,88 @@ std::vector<Chain> Vectorizer::splitChainByAlignment(Chain &C) {
return Ret;
}
+namespace {
+
+/// Reinterpret \p V as \p NewTy when both have the same total size. LLVM uses
+/// \p bitcast as the opcode for "bytecast"—same-width casts involving \p bN or
+/// byte vectors—so pointers do not unnecessarily go through \p ptrtoint /
+/// \p inttoptr. Use \p CreateBitOrPointerCast when neither type is involved in
+/// that encoding or when integer-to-pointer (\p inttoptr) is required.
+Value *reinterpretToSameSizedType(IRBuilderBase &Builder, Value *V,
+ Type *NewTy) {
+ Type *VTy = V->getType();
+ if ((VTy->isByteOrByteVectorTy() &&
+ (NewTy->isPtrOrPtrVectorTy() || NewTy->isIntOrIntVectorTy() ||
+ NewTy->isFPOrFPVectorTy())) ||
+ (NewTy->isByteOrByteVectorTy() &&
+ (VTy->isPtrOrPtrVectorTy() || VTy->isIntOrIntVectorTy() ||
+ VTy->isFPOrFPVectorTy())))
+ return Builder.CreateBitCast(V, NewTy);
+ return Builder.CreateBitOrPointerCast(V, NewTy);
+}
+
+/// View \p V as a scalar integer \p iSrcBits wide, using \p bitcast to flatten
+/// vectors and to cross FP / byte / int representations without resizing.
+Value *flattenToScalarInt(IRBuilderBase &Builder, Value *V, unsigned SrcBits,
+ LLVMContext &Ctx) {
+ Type *VTy = V->getType();
+ auto *IntTy = Type::getIntNTy(Ctx, SrcBits);
+ if (VTy->isVectorTy())
+ return Builder.CreateBitCast(V, IntTy);
+ if (VTy->isIntegerTy())
+ return V;
+ if (VTy->isByteTy())
+ return Builder.CreateBitCast(V, IntTy);
+ if (VTy->isFloatingPointTy())
+ return Builder.CreateBitCast(V, IntTy);
+ assert(VTy->isPointerTy() && "Unexpected type when flattening to integer");
+ return Builder.CreatePtrToInt(V, IntTy);
+}
+
+} // namespace
+
+/// Reinterpret the memory bit pattern between normalized vector lane types
+/// (\p bN or \p iN for sub-byte chains) and original load/store types.
+///
+/// Pipeline: normalize \p <N x i1> lanes to \p b1, flatten to \p iSrcBits when
+/// widths differ (\p bitcast for "bags of bits"), \p zext/\p trunc to the
+/// destination width, then \p reinterpretToSameSizedType (bitcast-first for
+/// byte types).
+Value *Vectorizer::resizeAndBitcast(Value *V, Type *NewTy) {
+ Type *VTy = V->getType();
+ assert(VTy != NewTy && "Types should mismatch in this function.");
+ assert(!(VTy->isFPOrFPVectorTy() && NewTy->isPtrOrPtrVectorTy()) &&
+ "FP->Ptr cast should be unreachable: vector lanes are bN/iN before "
+ "reinterpret.");
+ assert(!(VTy->isPtrOrPtrVectorTy() && NewTy->isFPOrFPVectorTy()) &&
+ "Ptr->FP cast should be unreachable: vector lanes are bN/iN before "
+ "reinterpret.");
+
+ LLVMContext &Ctx = F.getContext();
+
+ // Normalize <N x i1> to b1 lanes (LLVM uses bitcast for i1→byte widening).
+ if (auto *VecTy = dyn_cast<VectorType>(VTy);
+ VecTy && VecTy->getElementType()->isIntegerTy(1)) {
+ Type *ByteVecTy = VecTy->getWithNewType(Type::getByteNTy(Ctx, 1));
+ V = Builder.CreateBitCast(V, ByteVecTy);
+ VTy = V->getType();
+ }
+
+ unsigned VTySize = DL.getTypeSizeInBits(VTy);
+ unsigned NewTySize = DL.getTypeSizeInBits(NewTy);
+
+ if (VTySize != NewTySize) {
+ assert(!VTy->isPtrOrPtrVectorTy() &&
+ "Pointer values should never have mismatched bitwidths: "
+ "wide pointers use b<GCD> lanes that pack to native ptr size.");
+ V = flattenToScalarInt(Builder, V, VTySize, Ctx);
+ V = Builder.CreateZExtOrTrunc(V, Type::getIntNTy(Ctx, NewTySize));
+ return reinterpretToSameSizedType(Builder, V, NewTy);
+ }
+
+ return reinterpretToSameSizedType(Builder, V, NewTy);
+}
+
bool Vectorizer::vectorizeChain(Chain &C) {
if (C.size() < 2)
return false;
@@ -1094,7 +1200,11 @@ bool Vectorizer::vectorizeChain(Chain &C) {
dumpChain(C);
});
- Type *VecElemTy = getChainElemTy(C);
+ // VecElemTy is the normalized GCD lane type: b<GCD> for GCD >= 8, else i<GCD>
+ // for sub-byte chains. Original scalar types are recovered via
+ // resizeAndBitcast after extract/shuffle.
+ Type *VecElemTy = C.computeAndCacheElemTy(F.getContext(), DL);
+ unsigned VecElemSize = DL.getTypeSizeInBits(VecElemTy);
bool IsLoadChain = isa<LoadInst>(C[0].Inst);
unsigned AS = getLoadStoreAddressSpace(C[0].Inst);
unsigned BytesAdded = DL.getTypeStoreSize(getLoadStoreType(&*C[0].Inst));
@@ -1110,10 +1220,10 @@ bool Vectorizer::vectorizeChain(Chain &C) {
PrevReadEnd = APIntOps::smax(PrevReadEnd, ReadEnd);
}
- assert(8 * ChainBytes % DL.getTypeSizeInBits(VecElemTy) == 0);
- // VecTy is a power of 2 and 1 byte at smallest, but VecElemTy may be smaller
- // than 1 byte (e.g. VecTy == <32 x i1>).
- unsigned NumElem = 8 * ChainBytes / DL.getTypeSizeInBits(VecElemTy);
+ assert(8 * ChainBytes % VecElemSize == 0 &&
+ "ChainBytes must be a multiple of VecElemTy size");
+
+ unsigned NumElem = 8 * ChainBytes / VecElemSize;
Type *VecTy = FixedVectorType::get(VecElemTy, NumElem);
Align Alignment = getLoadStoreAlignment(C[0].Inst);
@@ -1126,11 +1236,16 @@ bool Vectorizer::vectorizeChain(Chain &C) {
MaybeAlign(), DL, C[0].Inst, nullptr, &DT));
}
- // All elements of the chain must have the same scalar-type size.
#ifndef NDEBUG
- for (const ChainElem &E : C)
- assert(DL.getTypeStoreSize(getLoadStoreType(E.Inst)->getScalarType()) ==
- DL.getTypeStoreSize(VecElemTy));
+ // Each element's scalar size must be a whole multiple of VecElemSize so that
+ // it occupies an integer number of contiguous vector lanes.
+ for (const ChainElem &E : C) {
+ unsigned ElemTySize = DL.getTypeSizeInBits(getLoadStoreType(E.Inst));
+ assert(ElemTySize % VecElemSize == 0 &&
+ "Element size is not a multiple of"
+ " the vector element size, which would break the vectorization "
+ "layout.");
+ }
#endif
Instruction *VecInst;
@@ -1161,26 +1276,55 @@ bool Vectorizer::vectorizeChain(Chain &C) {
VecTy, getLoadStorePointerOperand(C[0].Inst), Alignment);
}
+ // Replace each original load with the appropriate slice of the vector
+ // result. The extraction strategy depends on how many vector lanes the
+ // original type occupies:
+ // - scalar result (VecTy is not a vector): use the vector directly.
+ // - original type is a vector: shufflevector to extract the sub-range.
+ // - original type maps to multiple lanes (e.g. i32 in a <4 x i16>
+ // vector): shufflevector to collect those lanes.
+ // - original type maps to a single lane: extractelement.
+ // After extraction, resizeAndBitcast handles any remaining type mismatch
+ // (e.g. converting a <2 x i16> shuffle result back to i32).
for (const ChainElem &E : C) {
Instruction *I = E.Inst;
- Value *V;
+ Value *ExtractedVal;
Type *T = getLoadStoreType(I);
+ unsigned NumElems = DL.getTypeSizeInBits(T) / VecElemSize;
+
+ // Byte offset of this element relative to the chain base (C[0]).
unsigned EOffset =
(E.OffsetFromLeader - C[0].OffsetFromLeader).getZExtValue();
- unsigned VecIdx = 8 * EOffset / DL.getTypeSizeInBits(VecElemTy);
+
+ // Convert the byte offset to a lane index within VecTy.
+ unsigned VecIdx = 8 * EOffset / VecElemSize;
if (!VecTy->isVectorTy()) {
- V = VecInst;
- } else if (auto *VT = dyn_cast<FixedVectorType>(T)) {
- auto Mask = llvm::to_vector<8>(
- llvm::seq<int>(VecIdx, VecIdx + VT->getNumElements()));
- V = Builder.CreateShuffleVector(VecInst, Mask, I->getName());
+ // Degenerate case: the whole chain collapsed to a single scalar load.
+ ExtractedVal = VecInst;
+ } else if (NumElems > 1) {
+ // T spans multiple VecElemTy-sized lanes — either because T is a vector
+ // type or a scalar wider than VecElemTy (e.g. i32 in a <N x i16>
+ // vector). In both cases, collect the NumElems lanes starting at
+ // VecIdx; resizeAndBitcast below will fix up any remaining type
+ // mismatch.
+ auto Mask =
+ llvm::to_vector<8>(llvm::seq<int>(VecIdx, VecIdx + NumElems));
+ ExtractedVal = Builder.CreateShuffleVector(VecInst, Mask, I->getName());
} else {
- V = Builder.CreateExtractElement(VecInst, Builder.getInt32(VecIdx),
- I->getName());
+ // Single-lane element; a simple extractelement suffices.
+ ExtractedVal = Builder.CreateExtractElement(
+ VecInst, Builder.getInt32(VecIdx), I->getName());
}
- if (V->getType() != I->getType())
- V = Builder.CreateBitOrPointerCast(V, I->getType());
- I->replaceAllUsesWith(V);
+
+ // If the extracted value's type doesn't exactly match the original load
+ // type (possible after mixed-type vectorization), cast it back.
+ Type *ExpectedTy = getLoadStoreType(I);
+ if (ExtractedVal->getType() != ExpectedTy) {
+ LLVM_DEBUG(dbgs() << "LSV: Casting " << *ExtractedVal << " to "
+ << *ExpectedTy << '\n');
+ ExtractedVal = resizeAndBitcast(ExtractedVal, ExpectedTy);
+ }
+ I->replaceAllUsesWith(ExtractedVal);
}
// Finally, we need to reorder the instrs in the BB so that the (transitive)
@@ -1209,27 +1353,44 @@ bool Vectorizer::vectorizeChain(Chain &C) {
return A.Inst->comesBefore(B.Inst);
})->Inst);
- // Build the vector to store.
+ // Build the vector value to store by inserting each scalar operand into
+ // its corresponding lane(s). Start with a poison vector so that any
+ // lanes not covered by the chain have defined (if undefined) values.
Value *Vec = PoisonValue::get(VecTy);
- auto InsertElem = [&](Value *V, unsigned VecIdx) {
- if (V->getType() != VecElemTy)
- V = Builder.CreateBitOrPointerCast(V, VecElemTy);
- Vec = Builder.CreateInsertElement(Vec, V, Builder.getInt32(VecIdx));
- };
+
for (const ChainElem &E : C) {
auto *I = cast<StoreInst>(E.Inst);
+ Value *Val = I->getValueOperand();
+
+ // Number of VecElemTy-sized lanes that this store value occupies.
+ unsigned NumElems = DL.getTypeSizeInBits(Val->getType()) / VecElemSize;
+
+ // Derive VecIdx from the element's offset so that overlapping elements
+ // are inserted at the correct lane rather than one past the previous end.
unsigned EOffset =
(E.OffsetFromLeader - C[0].OffsetFromLeader).getZExtValue();
- unsigned VecIdx = 8 * EOffset / DL.getTypeSizeInBits(VecElemTy);
- if (FixedVectorType *VT =
- dyn_cast<FixedVectorType>(getLoadStoreType(I))) {
- for (int J = 0, JE = VT->getNumElements(); J < JE; ++J) {
- InsertElem(Builder.CreateExtractElement(I->getValueOperand(),
- Builder.getInt32(J)),
- VecIdx++);
- }
+ unsigned VecIdx = 8 * EOffset / VecElemSize;
+
+ Value *CastVal = Val;
+ if (NumElems == 1) {
+ // Single-lane value: cast to VecElemTy if needed, then insert.
+ if (CastVal->getType() != VecElemTy)
+ CastVal = resizeAndBitcast(CastVal, VecElemTy);
+ Vec =
+ Builder.CreateInsertElement(Vec, CastVal, Builder.getInt32(VecIdx));
} else {
- InsertElem(I->getValueOperand(), VecIdx);
+ // Multi-lane value (e.g. storing an i32 into a <N x i16> vector):
+ // reinterpret as a sub-vector of VecElemTy, then scatter each lane.
+ Type *SubVecTy = FixedVectorType::get(VecElemTy, NumElems);
+ if (CastVal->getType() != SubVecTy)
+ CastVal = resizeAndBitcast(CastVal, SubVecTy);
+
+ for (unsigned ElmIdx = 0; ElmIdx < NumElems; ++ElmIdx) {
+ Value *Elt =
+ Builder.CreateExtractElement(CastVal, Builder.getInt32(ElmIdx));
+ Vec = Builder.CreateInsertElement(Vec, Elt,
+ Builder.getInt32(VecIdx + ElmIdx));
+ }
}
}
@@ -1250,8 +1411,11 @@ bool Vectorizer::vectorizeChain(Chain &C) {
}
}
+ // Copy metadata (e.g. !tbaa, !alias.scope) from the original instructions
+ // to the new vectorized instruction.
propagateMetadata(VecInst, C);
+ // Mark the original scalar instructions for removal.
for (const ChainElem &E : C)
ToErase.emplace_back(E.Inst);
@@ -1607,13 +1771,12 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
return;
// The reduced key has all elements of the ECClassKey except the underlying
- // object. Check that EqClassKey has 4 elements and define the reduced key.
- static_assert(std::tuple_size_v<EqClassKey> == 4,
+ // object. Check that EqClassKey has 3 elements and define the reduced key.
+ static_assert(std::tuple_size_v<EqClassKey> == 3,
"EqClassKey has changed - EqClassReducedKey needs changes too");
using EqClassReducedKey =
std::tuple<std::tuple_element_t<1, EqClassKey> /* AddrSpace */,
- std::tuple_element_t<2, EqClassKey> /* Element size */,
- std::tuple_element_t<3, EqClassKey> /* IsLoad; */>;
+ std::tuple_element_t<2, EqClassKey> /* IsLoad; */>;
using ECReducedKeyToUnderlyingObjectMap =
MapVector<EqClassReducedKey,
SmallPtrSet<std::tuple_element_t<0, EqClassKey>, 4>>;
@@ -1625,8 +1788,7 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
bool FoundPotentiallyOptimizableEC = false;
for (const auto &EC : EQClasses) {
const auto &Key = EC.first;
- EqClassReducedKey RedKey{std::get<1>(Key), std::get<2>(Key),
- std::get<3>(Key)};
+ EqClassReducedKey RedKey{std::get<1>(Key), std::get<2>(Key)};
auto &UOMap = RedKeyToUOMap[RedKey];
UOMap.insert(std::get<0>(Key));
if (UOMap.size() > 1)
@@ -1647,8 +1809,7 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
dbgs() << "LSV: mergeEquivalenceClasses: RedKeyToUOMap:\n";
for (const auto &RedKeyToUO : RedKeyToUOMap) {
dbgs() << " Reduced key: {" << std::get<0>(RedKeyToUO.first) << ", "
- << std::get<1>(RedKeyToUO.first) << ", "
- << static_cast<int>(std::get<2>(RedKeyToUO.first)) << "} --> "
+ << static_cast<int>(std::get<1>(RedKeyToUO.first)) << "} --> "
<< RedKeyToUO.second.size() << " underlying objects:\n";
for (auto UObject : RedKeyToUO.second)
dbgs() << " " << *UObject << '\n';
@@ -1688,10 +1849,9 @@ void Vectorizer::mergeEquivalenceClasses(EquivalenceClassMap &EQClasses) const {
if (UObject == UltimateTarget)
continue;
- EqClassKey KeyFrom{UObject, std::get<0>(RedKey), std::get<1>(RedKey),
- std::get<2>(RedKey)};
- EqClassKey KeyTo{UltimateTarget, std::get<0>(RedKey), std::get<1>(RedKey),
- std::get<2>(RedKey)};
+ EqClassKey KeyFrom{UObject, std::get<0>(RedKey), std::get<1>(RedKey)};
+ EqClassKey KeyTo{UltimateTarget, std::get<0>(RedKey),
+ std::get<1>(RedKey)};
// The entry for KeyFrom is guarantted to exist, unlike KeyTo. Thus,
// request the reference to the instructions vector for KeyTo first.
const auto &VecTo = EQClasses[KeyTo];
@@ -1783,7 +1943,6 @@ Vectorizer::collectEquivalenceClasses(BasicBlock::iterator Begin,
continue;
Ret[{GetUnderlyingObject(Ptr), AS,
- DL.getTypeSizeInBits(getLoadStoreType(&I)->getScalarType()),
/*IsLoad=*/LI != nullptr}]
.emplace_back(&I);
}
@@ -1997,6 +2156,8 @@ Value *Vectorizer::createMaskForExtraElements(const ArrayRef<ChainElem> C,
// Start each mask element as false
SmallVector<Constant *, 64> MaskElts(VecTy->getNumElements(),
Builder.getInt1(false));
+ unsigned ScalarTySize = DL.getTypeSizeInBits(VecTy->getScalarType());
+
// Iterate over the chain and set the corresponding mask element to true for
// each element that is not an extra element.
for (const ChainElem &E : C) {
@@ -2004,14 +2165,10 @@ Value *Vectorizer::createMaskForExtraElements(const ArrayRef<ChainElem> C,
continue;
unsigned EOffset =
(E.OffsetFromLeader - C[0].OffsetFromLeader).getZExtValue();
- unsigned VecIdx =
- 8 * EOffset / DL.getTypeSizeInBits(VecTy->getScalarType());
- if (FixedVectorType *VT =
- dyn_cast<FixedVectorType>(getLoadStoreType(E.Inst)))
- for (unsigned J = 0; J < VT->getNumElements(); ++J)
- MaskElts[VecIdx + J] = Builder.getInt1(true);
- else
- MaskElts[VecIdx] = Builder.getInt1(true);
+ unsigned ElemSize = DL.getTypeSizeInBits(getLoadStoreType(E.Inst));
+ unsigned VecIdx = 8 * EOffset / ScalarTySize;
+ unsigned NumElems = ElemSize / ScalarTySize;
+ std::fill_n(MaskElts.begin() + VecIdx, NumElems, Builder.getInt1(true));
}
return ConstantVector::get(MaskElts);
}
diff --git a/llvm/test/CodeGen/AMDGPU/32-bit-local-address-space.ll b/llvm/test/CodeGen/AMDGPU/32-bit-local-address-space.ll
index 4b53f66b379a4..1c05a49bbde54 100644
--- a/llvm/test/CodeGen/AMDGPU/32-bit-local-address-space.ll
+++ b/llvm/test/CodeGen/AMDGPU/32-bit-local-address-space.ll
@@ -14,11 +14,10 @@
define amdgpu_kernel void @local_address_load(ptr addrspace(1) %out, ptr addrspace(3) %in) {
; GFX7-LABEL: local_address_load:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dword s2, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: v_mov_b32_e32 v0, s2
; GFX7-NEXT: ds_read_b32 v0, v0
; GFX7-NEXT: s_mov_b32 s2, -1
@@ -28,11 +27,10 @@ define amdgpu_kernel void @local_address_load(ptr addrspace(1) %out, ptr addrspa
;
; GFX8-LABEL: local_address_load:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: ds_read_b32 v0, v0
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -85,11 +83,10 @@ entry:
define amdgpu_kernel void @local_address_gep_const_offset(ptr addrspace(1) %out, ptr addrspace(3) %in) {
; GFX7-LABEL: local_address_gep_const_offset:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dword s2, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: v_mov_b32_e32 v0, s2
; GFX7-NEXT: ds_read_b32 v0, v0 offset:4
; GFX7-NEXT: s_mov_b32 s2, -1
@@ -99,11 +96,10 @@ define amdgpu_kernel void @local_address_gep_const_offset(ptr addrspace(1) %out,
;
; GFX8-LABEL: local_address_gep_const_offset:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: ds_read_b32 v0, v0 offset:4
; GFX8-NEXT: s_mov_b32 s2, -1
@@ -121,11 +117,10 @@ entry:
define amdgpu_kernel void @local_address_gep_large_const_offset(ptr addrspace(1) %out, ptr addrspace(3) %in) {
; GFX7-LABEL: local_address_gep_large_const_offset:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dword s2, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
; GFX7-NEXT: s_add_i32 s2, s2, 0x10004
; GFX7-NEXT: v_mov_b32_e32 v0, s2
; GFX7-NEXT: ds_read_b32 v0, v0
@@ -136,11 +131,10 @@ define amdgpu_kernel void @local_address_gep_large_const_offset(ptr addrspace(1)
;
; GFX8-LABEL: local_address_gep_large_const_offset:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
; GFX8-NEXT: s_add_i32 s2, s2, 0x10004
; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: ds_read_b32 v0, v0
@@ -158,30 +152,32 @@ entry:
define amdgpu_kernel void @null_32bit_lds_ptr(ptr addrspace(1) %out, ptr addrspace(3) %lds) nounwind {
; GFX7-LABEL: null_32bit_lds_ptr:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7-NEXT: s_movk_i32 s4, 0x7b
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_cmp_lg_u32 s6, 0
-; GFX7-NEXT: s_cselect_b32 s4, s4, 0x1c8
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_cmp_lg_u32 s2, 0
+; GFX7-NEXT: s_movk_i32 s0, 0x7b
+; GFX7-NEXT: s_cselect_b32 s0, s0, 0x1c8
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: null_32bit_lds_ptr:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_movk_i32 s4, 0x7b
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_cmp_lg_u32 s6, 0
-; GFX8-NEXT: s_cselect_b32 s4, s4, 0x1c8
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_cmp_lg_u32 s2, 0
+; GFX8-NEXT: s_movk_i32 s0, 0x7b
+; GFX8-NEXT: s_cselect_b32 s0, s0, 0x1c8
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
%cmp = icmp ne ptr addrspace(3) %lds, null
%x = select i1 %cmp, i32 123, i32 456
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_minmax.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_minmax.ll
index d233423e69135..fc6e74ee39245 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_minmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_minmax.ll
@@ -218,8 +218,7 @@ define amdgpu_kernel void @flat_atomic_min_ret_i64(ptr addrspace(1) %out, ptr %p
define amdgpu_kernel void @local_atomic_min_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; GFX9-LABEL: local_atomic_min_ret_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -231,12 +230,10 @@ define amdgpu_kernel void @local_atomic_min_ret_i32(ptr addrspace(1) %out, ptr a
;
; GFX10-LABEL: local_atomic_min_ret_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: ds_min_rtn_i32 v0, v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -246,11 +243,9 @@ define amdgpu_kernel void @local_atomic_min_ret_i32(ptr addrspace(1) %out, ptr a
;
; GFX11-LABEL: local_atomic_min_ret_i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: ds_min_rtn_i32 v0, v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -265,8 +260,7 @@ define amdgpu_kernel void @local_atomic_min_ret_i32(ptr addrspace(1) %out, ptr a
define amdgpu_kernel void @local_atomic_min_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; GFX9-LABEL: local_atomic_min_ret_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -279,13 +273,11 @@ define amdgpu_kernel void @local_atomic_min_ret_i64(ptr addrspace(1) %out, ptr a
;
; GFX10-LABEL: local_atomic_min_ret_i64:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: ds_min_rtn_i64 v[0:1], v2, v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -295,12 +287,10 @@ define amdgpu_kernel void @local_atomic_min_ret_i64(ptr addrspace(1) %out, ptr a
;
; GFX11-LABEL: local_atomic_min_ret_i64:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: ds_min_rtn_i64 v[0:1], v2, v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -516,8 +506,7 @@ define amdgpu_kernel void @flat_atomic_max_ret_i64(ptr addrspace(1) %out, ptr %p
define amdgpu_kernel void @local_atomic_max_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; GFX9-LABEL: local_atomic_max_ret_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -529,12 +518,10 @@ define amdgpu_kernel void @local_atomic_max_ret_i32(ptr addrspace(1) %out, ptr a
;
; GFX10-LABEL: local_atomic_max_ret_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: ds_max_rtn_i32 v0, v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -544,11 +531,9 @@ define amdgpu_kernel void @local_atomic_max_ret_i32(ptr addrspace(1) %out, ptr a
;
; GFX11-LABEL: local_atomic_max_ret_i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: ds_max_rtn_i32 v0, v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -563,8 +548,7 @@ define amdgpu_kernel void @local_atomic_max_ret_i32(ptr addrspace(1) %out, ptr a
define amdgpu_kernel void @local_atomic_max_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; GFX9-LABEL: local_atomic_max_ret_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -577,13 +561,11 @@ define amdgpu_kernel void @local_atomic_max_ret_i64(ptr addrspace(1) %out, ptr a
;
; GFX10-LABEL: local_atomic_max_ret_i64:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: ds_max_rtn_i64 v[0:1], v2, v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -593,12 +575,10 @@ define amdgpu_kernel void @local_atomic_max_ret_i64(ptr addrspace(1) %out, ptr a
;
; GFX11-LABEL: local_atomic_max_ret_i64:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: ds_max_rtn_i64 v[0:1], v2, v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -814,8 +794,7 @@ define amdgpu_kernel void @flat_atomic_umin_ret_i64(ptr addrspace(1) %out, ptr %
define amdgpu_kernel void @local_atomic_umin_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; GFX9-LABEL: local_atomic_umin_ret_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -827,12 +806,10 @@ define amdgpu_kernel void @local_atomic_umin_ret_i32(ptr addrspace(1) %out, ptr
;
; GFX10-LABEL: local_atomic_umin_ret_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: ds_min_rtn_u32 v0, v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -842,11 +819,9 @@ define amdgpu_kernel void @local_atomic_umin_ret_i32(ptr addrspace(1) %out, ptr
;
; GFX11-LABEL: local_atomic_umin_ret_i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: ds_min_rtn_u32 v0, v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -861,8 +836,7 @@ define amdgpu_kernel void @local_atomic_umin_ret_i32(ptr addrspace(1) %out, ptr
define amdgpu_kernel void @local_atomic_umin_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; GFX9-LABEL: local_atomic_umin_ret_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -875,13 +849,11 @@ define amdgpu_kernel void @local_atomic_umin_ret_i64(ptr addrspace(1) %out, ptr
;
; GFX10-LABEL: local_atomic_umin_ret_i64:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: ds_min_rtn_u64 v[0:1], v2, v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -891,12 +863,10 @@ define amdgpu_kernel void @local_atomic_umin_ret_i64(ptr addrspace(1) %out, ptr
;
; GFX11-LABEL: local_atomic_umin_ret_i64:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: ds_min_rtn_u64 v[0:1], v2, v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -1112,8 +1082,7 @@ define amdgpu_kernel void @flat_atomic_umax_ret_i64(ptr addrspace(1) %out, ptr %
define amdgpu_kernel void @local_atomic_umax_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; GFX9-LABEL: local_atomic_umax_ret_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -1125,12 +1094,10 @@ define amdgpu_kernel void @local_atomic_umax_ret_i32(ptr addrspace(1) %out, ptr
;
; GFX10-LABEL: local_atomic_umax_ret_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: ds_max_rtn_u32 v0, v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -1140,11 +1107,9 @@ define amdgpu_kernel void @local_atomic_umax_ret_i32(ptr addrspace(1) %out, ptr
;
; GFX11-LABEL: local_atomic_umax_ret_i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: ds_max_rtn_u32 v0, v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -1159,8 +1124,7 @@ define amdgpu_kernel void @local_atomic_umax_ret_i32(ptr addrspace(1) %out, ptr
define amdgpu_kernel void @local_atomic_umax_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; GFX9-LABEL: local_atomic_umax_ret_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1173,13 +1137,11 @@ define amdgpu_kernel void @local_atomic_umax_ret_i64(ptr addrspace(1) %out, ptr
;
; GFX10-LABEL: local_atomic_umax_ret_i64:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: ds_max_rtn_u64 v[0:1], v2, v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -1189,12 +1151,10 @@ define amdgpu_kernel void @local_atomic_umax_ret_i64(ptr addrspace(1) %out, ptr
;
; GFX11-LABEL: local_atomic_umax_ret_i64:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: ds_max_rtn_u64 v[0:1], v2, v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
index f0c8dae40280a..fcafaa0c9172e 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_udec_wrap.ll
@@ -16,17 +16,16 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0
define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_dec_ret_i32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_add_i32 s12, s12, s17
+; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: ds_dec_rtn_u32 v2, v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
@@ -34,17 +33,16 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr add
;
; VI-LABEL: lds_atomic_dec_ret_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_add_i32 s12, s12, s17
+; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: ds_dec_rtn_u32 v2, v1, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -52,8 +50,7 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr add
;
; GFX9-LABEL: lds_atomic_dec_ret_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -65,12 +62,10 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr add
;
; GFX10-LABEL: lds_atomic_dec_ret_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: ds_dec_rtn_u32 v0, v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -80,11 +75,9 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr add
;
; GFX11-LABEL: lds_atomic_dec_ret_i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: ds_dec_rtn_u32 v0, v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -99,17 +92,16 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr add
define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_dec_ret_i32_offset:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_add_i32 s12, s12, s17
+; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: ds_dec_rtn_u32 v2, v1, v0 offset:16
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
@@ -117,17 +109,16 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out,
;
; VI-LABEL: lds_atomic_dec_ret_i32_offset:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_add_i32 s12, s12, s17
+; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: ds_dec_rtn_u32 v2, v1, v0 offset:16
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -135,8 +126,7 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out,
;
; GFX9-LABEL: lds_atomic_dec_ret_i32_offset:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -148,12 +138,10 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out,
;
; GFX10-LABEL: lds_atomic_dec_ret_i32_offset:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
; GFX10-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -163,11 +151,9 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out,
;
; GFX11-LABEL: lds_atomic_dec_ret_i32_offset:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -2544,16 +2530,15 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0(ptr addrspace(1) %out, ptr
define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_dec_ret_i64:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -2563,16 +2548,15 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr add
;
; VI-LABEL: lds_atomic_dec_ret_i64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -2582,8 +2566,7 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr add
;
; GFX9-LABEL: lds_atomic_dec_ret_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2596,13 +2579,11 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr add
;
; GFX10-LABEL: lds_atomic_dec_ret_i64:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -2612,12 +2593,10 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr add
;
; GFX11-LABEL: lds_atomic_dec_ret_i64:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -2632,16 +2611,15 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr add
define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_dec_ret_i64_offset:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -2651,16 +2629,15 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out,
;
; VI-LABEL: lds_atomic_dec_ret_i64_offset:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -2670,8 +2647,7 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out,
;
; GFX9-LABEL: lds_atomic_dec_ret_i64_offset:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -2684,13 +2660,11 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out,
;
; GFX10-LABEL: lds_atomic_dec_ret_i64_offset:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -2700,12 +2674,10 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out,
;
; GFX11-LABEL: lds_atomic_dec_ret_i64_offset:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
index 62caf7f406325..2e6ca69b77f78 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/atomicrmw_uinc_wrap.ll
@@ -17,17 +17,16 @@ declare i32 @llvm.amdgcn.workitem.id.x() #0
define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_inc_ret_i32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_add_i32 s12, s12, s17
+; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: ds_inc_rtn_u32 v2, v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
@@ -35,17 +34,16 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr add
;
; VI-LABEL: lds_atomic_inc_ret_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_add_i32 s12, s12, s17
+; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: ds_inc_rtn_u32 v2, v1, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -53,8 +51,7 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr add
;
; GFX9-LABEL: lds_atomic_inc_ret_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -66,12 +63,10 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr add
;
; GFX10-LABEL: lds_atomic_inc_ret_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: ds_inc_rtn_u32 v0, v0, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -81,11 +76,9 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr add
;
; GFX11-LABEL: lds_atomic_inc_ret_i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v1, 42 :: v_dual_mov_b32 v0, s2
; GFX11-NEXT: ds_inc_rtn_u32 v0, v0, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -112,17 +105,16 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr add
define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_inc_ret_i32_offset:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_add_i32 s12, s12, s17
+; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: ds_inc_rtn_u32 v2, v1, v0 offset:16
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_dword v[0:1], v2
@@ -130,17 +122,16 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out,
;
; VI-LABEL: lds_atomic_inc_ret_i32_offset:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_add_i32 s12, s12, s17
+; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: ds_inc_rtn_u32 v2, v1, v0 offset:16
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_store_dword v[0:1], v2
@@ -148,8 +139,7 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out,
;
; GFX9-LABEL: lds_atomic_inc_ret_i32_offset:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -161,12 +151,10 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out,
;
; GFX10-LABEL: lds_atomic_inc_ret_i32_offset:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
; GFX10-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -176,11 +164,9 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out,
;
; GFX11-LABEL: lds_atomic_inc_ret_i32_offset:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -1188,16 +1174,15 @@ define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(ptr addrspace(1) %out,
define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_inc_ret_i64:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -1207,16 +1192,15 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr add
;
; VI-LABEL: lds_atomic_inc_ret_i64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -1226,8 +1210,7 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr add
;
; GFX9-LABEL: lds_atomic_inc_ret_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1240,13 +1223,11 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr add
;
; GFX10-LABEL: lds_atomic_inc_ret_i64:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -1256,12 +1237,10 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr add
;
; GFX11-LABEL: lds_atomic_inc_ret_i64:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -1289,16 +1268,15 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr add
define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #1 {
; CI-LABEL: lds_atomic_inc_ret_i64_offset:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -1308,16 +1286,15 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out,
;
; VI-LABEL: lds_atomic_inc_ret_i64_offset:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -1327,8 +1304,7 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out,
;
; GFX9-LABEL: lds_atomic_inc_ret_i64_offset:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1341,13 +1317,11 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out,
;
; GFX10-LABEL: lds_atomic_inc_ret_i64_offset:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 42
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
@@ -1357,12 +1331,10 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out,
;
; GFX11-LABEL: lds_atomic_inc_ret_i64_offset:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_dual_mov_b32 v0, 42 :: v_dual_mov_b32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
index 57068d59c3e32..097141f062aee 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/extractelement.ll
@@ -2891,8 +2891,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
; GPRIDX-NEXT: runtime_loader_kernel_symbol = 0
; GPRIDX-NEXT: .end_amd_kernel_code_t
; GPRIDX-NEXT: ; %bb.0: ; %entry
-; GPRIDX-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GPRIDX-NEXT: s_load_dword s2, s[8:9], 0x8
+; GPRIDX-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GPRIDX-NEXT: s_mov_b64 s[4:5], 1.0
; GPRIDX-NEXT: s_mov_b32 s12, 0
; GPRIDX-NEXT: s_mov_b32 s13, 0x40140000
@@ -2979,8 +2978,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
; MOVREL-NEXT: runtime_loader_kernel_symbol = 0
; MOVREL-NEXT: .end_amd_kernel_code_t
; MOVREL-NEXT: ; %bb.0: ; %entry
-; MOVREL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; MOVREL-NEXT: s_load_dword s2, s[8:9], 0x8
+; MOVREL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; MOVREL-NEXT: s_add_i32 s12, s12, s17
; MOVREL-NEXT: s_mov_b64 s[4:5], 1.0
; MOVREL-NEXT: s_mov_b32 flat_scratch_lo, s13
@@ -3055,7 +3053,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
; GFX10-NEXT: gds_segment_byte_size = 0
; GFX10-NEXT: kernarg_segment_byte_size = 28
; GFX10-NEXT: workgroup_fbarrier_count = 0
-; GFX10-NEXT: wavefront_sgpr_count = 13
+; GFX10-NEXT: wavefront_sgpr_count = 16
; GFX10-NEXT: workitem_vgpr_count = 3
; GFX10-NEXT: reserved_vgpr_first = 0
; GFX10-NEXT: reserved_vgpr_count = 0
@@ -3071,9 +3069,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
; GFX10-NEXT: runtime_loader_kernel_symbol = 0
; GFX10-NEXT: .end_amd_kernel_code_t
; GFX10-NEXT: ; %bb.0: ; %entry
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s12, s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[12:15], s[8:9], 0x0
; GFX10-NEXT: s_mov_b64 s[0:1], 1.0
; GFX10-NEXT: s_mov_b32 s8, 0
; GFX10-NEXT: s_mov_b32 s9, 0x40140000
@@ -3083,11 +3079,11 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
; GFX10-NEXT: s_mov_b64 s[2:3], 2.0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 m0, s12
+; GFX10-NEXT: s_mov_b32 m0, s14
; GFX10-NEXT: s_movrels_b64 s[0:1], s[0:1]
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[12:13]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: dyn_extract_v5f64_s_s:
@@ -3144,7 +3140,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
; GFX11-NEXT: gds_segment_byte_size = 0
; GFX11-NEXT: kernarg_segment_byte_size = 28
; GFX11-NEXT: workgroup_fbarrier_count = 0
-; GFX11-NEXT: wavefront_sgpr_count = 13
+; GFX11-NEXT: wavefront_sgpr_count = 16
; GFX11-NEXT: workitem_vgpr_count = 3
; GFX11-NEXT: reserved_vgpr_first = 0
; GFX11-NEXT: reserved_vgpr_count = 0
@@ -3160,9 +3156,7 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
; GFX11-NEXT: runtime_loader_kernel_symbol = 0
; GFX11-NEXT: .end_amd_kernel_code_t
; GFX11-NEXT: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s12, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[10:11], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[12:15], s[4:5], 0x0
; GFX11-NEXT: s_mov_b64 s[0:1], 1.0
; GFX11-NEXT: s_mov_b32 s8, 0
; GFX11-NEXT: s_mov_b32 s9, 0x40140000
@@ -3172,10 +3166,10 @@ define amdgpu_kernel void @dyn_extract_v5f64_s_s(ptr addrspace(1) %out, i32 %sel
; GFX11-NEXT: s_mov_b64 s[2:3], 2.0
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 m0, s12
+; GFX11-NEXT: s_mov_b32 m0, s14
; GFX11-NEXT: s_movrels_b64 s[0:1], s[0:1]
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[10:11]
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[12:13]
; GFX11-NEXT: s_endpgm
entry:
%ext = extractelement <5 x double> <double 1.0, double 2.0, double 3.0, double 4.0, double 5.0>, i32 %sel
@@ -3897,18 +3891,17 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
; GPRIDX-NEXT: runtime_loader_kernel_symbol = 0
; GPRIDX-NEXT: .end_amd_kernel_code_t
; GPRIDX-NEXT: ; %bb.0: ; %entry
-; GPRIDX-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
-; GPRIDX-NEXT: s_load_dword s6, s[8:9], 0x8
-; GPRIDX-NEXT: s_mov_b32 s0, 1.0
-; GPRIDX-NEXT: s_mov_b32 s3, 4.0
-; GPRIDX-NEXT: s_mov_b32 s2, 0x40400000
-; GPRIDX-NEXT: s_mov_b32 s1, 2.0
+; GPRIDX-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GPRIDX-NEXT: s_mov_b32 s4, 1.0
+; GPRIDX-NEXT: s_mov_b32 s7, 4.0
+; GPRIDX-NEXT: s_mov_b32 s6, 0x40400000
+; GPRIDX-NEXT: s_mov_b32 s5, 2.0
; GPRIDX-NEXT: s_waitcnt lgkmcnt(0)
-; GPRIDX-NEXT: s_mov_b32 m0, s6
+; GPRIDX-NEXT: s_mov_b32 m0, s2
; GPRIDX-NEXT: v_mov_b32_e32 v1, 0
-; GPRIDX-NEXT: s_movrels_b32 s0, s0
-; GPRIDX-NEXT: v_mov_b32_e32 v0, s0
-; GPRIDX-NEXT: global_store_dword v1, v0, s[4:5]
+; GPRIDX-NEXT: s_movrels_b32 s2, s4
+; GPRIDX-NEXT: v_mov_b32_e32 v0, s2
+; GPRIDX-NEXT: global_store_dword v1, v0, s[0:1]
; GPRIDX-NEXT: s_endpgm
;
; MOVREL-LABEL: dyn_extract_v4f32_s_s_s:
@@ -3981,21 +3974,20 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
; MOVREL-NEXT: runtime_loader_kernel_symbol = 0
; MOVREL-NEXT: .end_amd_kernel_code_t
; MOVREL-NEXT: ; %bb.0: ; %entry
-; MOVREL-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
-; MOVREL-NEXT: s_load_dword s6, s[8:9], 0x8
-; MOVREL-NEXT: s_mov_b32 s0, 1.0
+; MOVREL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; MOVREL-NEXT: s_mov_b32 s4, 1.0
; MOVREL-NEXT: s_add_i32 s12, s12, s17
-; MOVREL-NEXT: s_mov_b32 s3, 4.0
-; MOVREL-NEXT: s_mov_b32 s2, 0x40400000
+; MOVREL-NEXT: s_mov_b32 s7, 4.0
+; MOVREL-NEXT: s_mov_b32 s6, 0x40400000
; MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; MOVREL-NEXT: s_mov_b32 m0, s6
-; MOVREL-NEXT: s_mov_b32 s1, 2.0
-; MOVREL-NEXT: s_movrels_b32 s0, s0
-; MOVREL-NEXT: v_mov_b32_e32 v0, s4
+; MOVREL-NEXT: s_mov_b32 m0, s2
+; MOVREL-NEXT: s_mov_b32 s5, 2.0
+; MOVREL-NEXT: s_movrels_b32 s2, s4
+; MOVREL-NEXT: v_mov_b32_e32 v0, s0
; MOVREL-NEXT: s_mov_b32 flat_scratch_lo, s13
; MOVREL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; MOVREL-NEXT: v_mov_b32_e32 v2, s0
-; MOVREL-NEXT: v_mov_b32_e32 v1, s5
+; MOVREL-NEXT: v_mov_b32_e32 v2, s2
+; MOVREL-NEXT: v_mov_b32_e32 v1, s1
; MOVREL-NEXT: flat_store_dword v[0:1], v2
; MOVREL-NEXT: s_endpgm
;
@@ -4069,19 +4061,17 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
; GFX10-NEXT: runtime_loader_kernel_symbol = 0
; GFX10-NEXT: .end_amd_kernel_code_t
; GFX10-NEXT: ; %bb.0: ; %entry
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s6, s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
-; GFX10-NEXT: s_mov_b32 s0, 1.0
-; GFX10-NEXT: s_mov_b32 s3, 4.0
-; GFX10-NEXT: s_mov_b32 s2, 0x40400000
-; GFX10-NEXT: s_mov_b32 s1, 2.0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-NEXT: s_mov_b32 s4, 1.0
+; GFX10-NEXT: s_mov_b32 s7, 4.0
+; GFX10-NEXT: s_mov_b32 s6, 0x40400000
+; GFX10-NEXT: s_mov_b32 s5, 2.0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 m0, s6
-; GFX10-NEXT: s_movrels_b32 s0, s0
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
+; GFX10-NEXT: s_mov_b32 m0, s2
+; GFX10-NEXT: s_movrels_b32 s2, s4
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: dyn_extract_v4f32_s_s_s:
@@ -4138,7 +4128,7 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
; GFX11-NEXT: gds_segment_byte_size = 0
; GFX11-NEXT: kernarg_segment_byte_size = 28
; GFX11-NEXT: workgroup_fbarrier_count = 0
-; GFX11-NEXT: wavefront_sgpr_count = 7
+; GFX11-NEXT: wavefront_sgpr_count = 8
; GFX11-NEXT: workitem_vgpr_count = 2
; GFX11-NEXT: reserved_vgpr_first = 0
; GFX11-NEXT: reserved_vgpr_count = 0
@@ -4154,19 +4144,17 @@ define amdgpu_kernel void @dyn_extract_v4f32_s_s_s(ptr addrspace(1) %out, i32 %s
; GFX11-NEXT: runtime_loader_kernel_symbol = 0
; GFX11-NEXT: .end_amd_kernel_code_t
; GFX11-NEXT: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
-; GFX11-NEXT: s_mov_b32 s0, 1.0
-; GFX11-NEXT: s_mov_b32 s3, 4.0
-; GFX11-NEXT: s_mov_b32 s2, 0x40400000
-; GFX11-NEXT: s_mov_b32 s1, 2.0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT: s_mov_b32 s4, 1.0
+; GFX11-NEXT: s_mov_b32 s7, 4.0
+; GFX11-NEXT: s_mov_b32 s6, 0x40400000
+; GFX11-NEXT: s_mov_b32 s5, 2.0
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 m0, s6
-; GFX11-NEXT: s_movrels_b32 s0, s0
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: global_store_b32 v1, v0, s[4:5]
+; GFX11-NEXT: s_mov_b32 m0, s2
+; GFX11-NEXT: s_movrels_b32 s2, s4
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT: s_endpgm
entry:
%ext = extractelement <4 x float> <float 1.0, float 2.0, float 3.0, float 4.0>, i32 %sel
@@ -4229,7 +4217,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
; GPRIDX-NEXT: gds_segment_byte_size = 0
; GPRIDX-NEXT: kernarg_segment_byte_size = 28
; GPRIDX-NEXT: workgroup_fbarrier_count = 0
-; GPRIDX-NEXT: wavefront_sgpr_count = 19
+; GPRIDX-NEXT: wavefront_sgpr_count = 18
; GPRIDX-NEXT: workitem_vgpr_count = 3
; GPRIDX-NEXT: reserved_vgpr_first = 0
; GPRIDX-NEXT: reserved_vgpr_count = 0
@@ -4245,20 +4233,19 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
; GPRIDX-NEXT: runtime_loader_kernel_symbol = 0
; GPRIDX-NEXT: .end_amd_kernel_code_t
; GPRIDX-NEXT: ; %bb.0: ; %entry
-; GPRIDX-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x0
-; GPRIDX-NEXT: s_load_dword s12, s[8:9], 0x8
+; GPRIDX-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
; GPRIDX-NEXT: s_mov_b64 s[0:1], 1.0
; GPRIDX-NEXT: s_mov_b64 s[6:7], 4.0
; GPRIDX-NEXT: s_mov_b32 s4, 0
; GPRIDX-NEXT: s_mov_b32 s5, 0x40080000
; GPRIDX-NEXT: s_waitcnt lgkmcnt(0)
-; GPRIDX-NEXT: s_mov_b32 m0, s12
+; GPRIDX-NEXT: s_mov_b32 m0, s10
; GPRIDX-NEXT: s_mov_b64 s[2:3], 2.0
; GPRIDX-NEXT: s_movrels_b64 s[0:1], s[0:1]
; GPRIDX-NEXT: v_mov_b32_e32 v0, s0
; GPRIDX-NEXT: v_mov_b32_e32 v1, s1
; GPRIDX-NEXT: v_mov_b32_e32 v2, 0
-; GPRIDX-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]
+; GPRIDX-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
; GPRIDX-NEXT: s_endpgm
;
; MOVREL-LABEL: dyn_extract_v4f64_s_s_s:
@@ -4331,23 +4318,22 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
; MOVREL-NEXT: runtime_loader_kernel_symbol = 0
; MOVREL-NEXT: .end_amd_kernel_code_t
; MOVREL-NEXT: ; %bb.0: ; %entry
-; MOVREL-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x0
-; MOVREL-NEXT: s_load_dword s8, s[8:9], 0x8
+; MOVREL-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
; MOVREL-NEXT: s_mov_b64 s[0:1], 1.0
; MOVREL-NEXT: s_mov_b64 s[6:7], 4.0
; MOVREL-NEXT: s_mov_b32 s4, 0
; MOVREL-NEXT: s_mov_b32 s5, 0x40080000
; MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; MOVREL-NEXT: s_mov_b32 m0, s8
+; MOVREL-NEXT: s_mov_b32 m0, s10
; MOVREL-NEXT: s_mov_b64 s[2:3], 2.0
; MOVREL-NEXT: s_movrels_b64 s[0:1], s[0:1]
; MOVREL-NEXT: s_add_i32 s12, s12, s17
; MOVREL-NEXT: v_mov_b32_e32 v0, s0
-; MOVREL-NEXT: v_mov_b32_e32 v2, s10
+; MOVREL-NEXT: v_mov_b32_e32 v2, s8
; MOVREL-NEXT: s_mov_b32 flat_scratch_lo, s13
; MOVREL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; MOVREL-NEXT: v_mov_b32_e32 v1, s1
-; MOVREL-NEXT: v_mov_b32_e32 v3, s11
+; MOVREL-NEXT: v_mov_b32_e32 v3, s9
; MOVREL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; MOVREL-NEXT: s_endpgm
;
@@ -4405,7 +4391,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
; GFX10-NEXT: gds_segment_byte_size = 0
; GFX10-NEXT: kernarg_segment_byte_size = 28
; GFX10-NEXT: workgroup_fbarrier_count = 0
-; GFX10-NEXT: wavefront_sgpr_count = 13
+; GFX10-NEXT: wavefront_sgpr_count = 12
; GFX10-NEXT: workitem_vgpr_count = 3
; GFX10-NEXT: reserved_vgpr_first = 0
; GFX10-NEXT: reserved_vgpr_count = 0
@@ -4421,9 +4407,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
; GFX10-NEXT: runtime_loader_kernel_symbol = 0
; GFX10-NEXT: .end_amd_kernel_code_t
; GFX10-NEXT: ; %bb.0: ; %entry
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s12, s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
; GFX10-NEXT: s_mov_b64 s[0:1], 1.0
; GFX10-NEXT: s_mov_b64 s[6:7], 4.0
; GFX10-NEXT: s_mov_b32 s4, 0
@@ -4431,11 +4415,11 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
; GFX10-NEXT: s_mov_b64 s[2:3], 2.0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 m0, s12
+; GFX10-NEXT: s_mov_b32 m0, s10
; GFX10-NEXT: s_movrels_b64 s[0:1], s[0:1]
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[10:11]
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: dyn_extract_v4f64_s_s_s:
@@ -4492,7 +4476,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
; GFX11-NEXT: gds_segment_byte_size = 0
; GFX11-NEXT: kernarg_segment_byte_size = 28
; GFX11-NEXT: workgroup_fbarrier_count = 0
-; GFX11-NEXT: wavefront_sgpr_count = 11
+; GFX11-NEXT: wavefront_sgpr_count = 12
; GFX11-NEXT: workitem_vgpr_count = 3
; GFX11-NEXT: reserved_vgpr_first = 0
; GFX11-NEXT: reserved_vgpr_count = 0
@@ -4508,9 +4492,7 @@ define amdgpu_kernel void @dyn_extract_v4f64_s_s_s(ptr addrspace(1) %out, i32 %s
; GFX11-NEXT: runtime_loader_kernel_symbol = 0
; GFX11-NEXT: .end_amd_kernel_code_t
; GFX11-NEXT: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s10, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[8:9], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[8:11], s[4:5], 0x0
; GFX11-NEXT: s_mov_b64 s[0:1], 1.0
; GFX11-NEXT: s_mov_b64 s[6:7], 4.0
; GFX11-NEXT: s_mov_b32 s4, 0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll
index 6573088a41fc2..c067bf64de240 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-call-return-values.ll
@@ -2955,12 +2955,14 @@ define amdgpu_kernel void @test_call_external_v33i32_func_v33i32_i32(ptr addrspa
; GCN-NEXT: [[COPY7:%[0-9]+]]:sgpr_64 = COPY $sgpr6_sgpr7
; GCN-NEXT: [[COPY8:%[0-9]+]]:sgpr_64 = COPY $sgpr4_sgpr5
; GCN-NEXT: [[COPY9:%[0-9]+]]:_(p4) = COPY $sgpr8_sgpr9
- ; GCN-NEXT: [[DEF:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
+ ; GCN-NEXT: [[DEF:%[0-9]+]]:_(<3 x s32>) = G_IMPLICIT_DEF
+ ; GCN-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 2
+ ; GCN-NEXT: [[DEF1:%[0-9]+]]:_(p1) = G_IMPLICIT_DEF
; GCN-NEXT: [[INT:%[0-9]+]]:_(p4) = G_INTRINSIC intrinsic(@llvm.amdgcn.kernarg.segment.ptr)
- ; GCN-NEXT: [[LOAD:%[0-9]+]]:_(p1) = G_LOAD [[INT]](p4) :: (dereferenceable invariant load (p1) from %ir.p.kernarg.offset1, align 16, addrspace 4)
- ; GCN-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8
- ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:_(p4) = nuw nusw inbounds G_PTR_ADD [[INT]], [[C]](s64)
- ; GCN-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[PTR_ADD]](p4) :: (dereferenceable invariant load (s32) from %ir.idx.kernarg.offset, align 8, addrspace 4)
+ ; GCN-NEXT: [[LOAD:%[0-9]+]]:_(<3 x s32>) = G_LOAD [[INT]](p4) :: (dereferenceable invariant load (<3 x s32>) from %ir.p.kernarg.offset1, align 16, addrspace 4)
+ ; GCN-NEXT: [[SHUF:%[0-9]+]]:_(<2 x s32>) = G_SHUFFLE_VECTOR [[LOAD]](<3 x s32>), [[DEF]], shufflemask(0, 1)
+ ; GCN-NEXT: [[BITCAST:%[0-9]+]]:_(p1) = G_BITCAST [[SHUF]](<2 x s32>)
+ ; GCN-NEXT: [[EVEC:%[0-9]+]]:_(s32) = G_EXTRACT_VECTOR_ELT [[LOAD]](<3 x s32>), [[C]](s32)
; GCN-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p5) = G_FRAME_INDEX %stack.0
; GCN-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $scc
; GCN-NEXT: [[GV:%[0-9]+]]:_(p0) = G_GLOBAL_VALUE @external_v33i32_func_v33i32_i32
@@ -2968,12 +2970,12 @@ define amdgpu_kernel void @test_call_external_v33i32_func_v33i32_i32(ptr addrspa
; GCN-NEXT: [[COPY11:%[0-9]+]]:_(p4) = COPY [[COPY7]]
; GCN-NEXT: [[COPY12:%[0-9]+]]:_(p4) = COPY [[COPY9]](p4)
; GCN-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 16
- ; GCN-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p4) = nuw inbounds G_PTR_ADD [[COPY12]], [[C1]](s64)
+ ; GCN-NEXT: [[PTR_ADD:%[0-9]+]]:_(p4) = nuw inbounds G_PTR_ADD [[COPY12]], [[C1]](s64)
; GCN-NEXT: [[COPY13:%[0-9]+]]:_(s64) = COPY [[COPY6]]
; GCN-NEXT: [[COPY14:%[0-9]+]]:_(s32) = COPY [[COPY5]]
; GCN-NEXT: [[COPY15:%[0-9]+]]:_(s32) = COPY [[COPY4]]
; GCN-NEXT: [[COPY16:%[0-9]+]]:_(s32) = COPY [[COPY3]]
- ; GCN-NEXT: [[DEF1:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
+ ; GCN-NEXT: [[DEF2:%[0-9]+]]:_(s32) = G_IMPLICIT_DEF
; GCN-NEXT: [[COPY17:%[0-9]+]]:_(s32) = COPY [[COPY2]](s32)
; GCN-NEXT: [[COPY18:%[0-9]+]]:_(s32) = COPY [[COPY1]](s32)
; GCN-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 10
@@ -2983,26 +2985,26 @@ define amdgpu_kernel void @test_call_external_v33i32_func_v33i32_i32(ptr addrspa
; GCN-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 20
; GCN-NEXT: [[SHL1:%[0-9]+]]:_(s32) = G_SHL [[COPY19]], [[C3]](s32)
; GCN-NEXT: [[OR1:%[0-9]+]]:_(s32) = G_OR [[OR]], [[SHL1]]
- ; GCN-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD]](p1)
+ ; GCN-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[BITCAST]](p1)
; GCN-NEXT: $vgpr0 = COPY [[FRAME_INDEX]](p5)
; GCN-NEXT: $vgpr1 = COPY [[UV]](s32)
; GCN-NEXT: $vgpr2 = COPY [[UV1]](s32)
- ; GCN-NEXT: $vgpr3 = COPY [[LOAD1]](s32)
+ ; GCN-NEXT: $vgpr3 = COPY [[EVEC]](s32)
; GCN-NEXT: [[COPY20:%[0-9]+]]:_(<4 x s32>) = COPY $private_rsrc_reg
; GCN-NEXT: $sgpr0_sgpr1_sgpr2_sgpr3 = COPY [[COPY20]](<4 x s32>)
; GCN-NEXT: $sgpr4_sgpr5 = COPY [[COPY10]](p4)
; GCN-NEXT: $sgpr6_sgpr7 = COPY [[COPY11]](p4)
- ; GCN-NEXT: $sgpr8_sgpr9 = COPY [[PTR_ADD1]](p4)
+ ; GCN-NEXT: $sgpr8_sgpr9 = COPY [[PTR_ADD]](p4)
; GCN-NEXT: $sgpr10_sgpr11 = COPY [[COPY13]](s64)
; GCN-NEXT: $sgpr12 = COPY [[COPY14]](s32)
; GCN-NEXT: $sgpr13 = COPY [[COPY15]](s32)
; GCN-NEXT: $sgpr14 = COPY [[COPY16]](s32)
- ; GCN-NEXT: $sgpr15 = COPY [[DEF1]](s32)
+ ; GCN-NEXT: $sgpr15 = COPY [[DEF2]](s32)
; GCN-NEXT: $vgpr31 = COPY [[OR1]](s32)
; GCN-NEXT: $sgpr30_sgpr31 = noconvergent G_SI_CALL [[GV]](p0), @external_v33i32_func_v33i32_i32, csr_amdgpu, implicit $vgpr0, implicit $vgpr1, implicit $vgpr2, implicit $vgpr3, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit $sgpr15, implicit $vgpr31
; GCN-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $scc
- ; GCN-NEXT: [[LOAD2:%[0-9]+]]:_(<33 x s32>) = G_LOAD [[FRAME_INDEX]](p5) :: (load (<33 x s32>) from %stack.0, align 256, addrspace 5)
- ; GCN-NEXT: G_STORE [[LOAD2]](<33 x s32>), [[DEF]](p1) :: (volatile store (<33 x s32>) into `ptr addrspace(1) poison`, align 8, addrspace 1)
+ ; GCN-NEXT: [[LOAD1:%[0-9]+]]:_(<33 x s32>) = G_LOAD [[FRAME_INDEX]](p5) :: (load (<33 x s32>) from %stack.0, align 256, addrspace 5)
+ ; GCN-NEXT: G_STORE [[LOAD1]](<33 x s32>), [[DEF1]](p1) :: (volatile store (<33 x s32>) into `ptr addrspace(1) poison`, align 8, addrspace 1)
; GCN-NEXT: S_ENDPGM 0
%val = call <33 x i32> @external_v33i32_func_v33i32_i32(ptr addrspace(1) %p, i32 %idx)
store volatile <33 x i32> %val, ptr addrspace(1) poison, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
index 8b5c27288453e..02e8f63395bad 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/legalize-bitcast.mir
@@ -218,7 +218,7 @@ body: |
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(p1) = G_BITCAST [[COPY]](p0)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](p1)
%0:_(p0) = COPY $vgpr0_vgpr1
- %1:_(p1) = G_BITCAST %0
+ %1:_(p1) = G_ADDRSPACE_CAST %0
$vgpr0_vgpr1 = COPY %1
...
@@ -235,7 +235,7 @@ body: |
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[COPY]](p1)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](p0)
%0:_(p1) = COPY $vgpr0_vgpr1
- %1:_(p0) = G_BITCAST %0
+ %1:_(p0) = G_ADDRSPACE_CAST %0
$vgpr0_vgpr1 = COPY %1
...
@@ -252,7 +252,7 @@ body: |
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(p0) = G_BITCAST [[COPY]](p999)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](p0)
%0:_(p999) = COPY $vgpr0_vgpr1
- %1:_(p0) = G_BITCAST %0
+ %1:_(p0) = G_ADDRSPACE_CAST %0
$vgpr0_vgpr1 = COPY %1
...
@@ -269,7 +269,7 @@ body: |
; CHECK-NEXT: [[BITCAST:%[0-9]+]]:_(p999) = G_BITCAST [[COPY]](p123)
; CHECK-NEXT: $vgpr0_vgpr1 = COPY [[BITCAST]](p999)
%0:_(p123) = COPY $vgpr0_vgpr1
- %1:_(p999) = G_BITCAST %0
+ %1:_(p999) = G_ADDRSPACE_CAST %0
$vgpr0_vgpr1 = COPY %1
...
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
index bdf54bd66c866..7e0e88b3af06d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.div.fmas.ll
@@ -480,14 +480,14 @@ define amdgpu_kernel void @test_div_fmas_f32_inline_imm_0(ptr addrspace(1) %out,
define amdgpu_kernel void @test_div_fmas_f32_inline_imm_1(ptr addrspace(1) %out, float %a, float %b, float %c, [8 x i32], i1 %d) {
; GFX7-LABEL: test_div_fmas_f32_inline_imm_1:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dword s2, s[4:5], 0x2
-; GFX7-NEXT: s_load_dword s3, s[4:5], 0x4
-; GFX7-NEXT: s_load_dword s6, s[4:5], 0xd
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_load_dword s3, s[4:5], 0x4
+; GFX7-NEXT: s_load_dword s4, s[4:5], 0xd
; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: s_cmp_lg_u32 s4, 0
; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: s_cmp_lg_u32 s6, 0
; GFX7-NEXT: s_cselect_b64 vcc, exec, 0
; GFX7-NEXT: v_div_fmas_f32 v0, v0, 1.0, v1
; GFX7-NEXT: s_mov_b32 s2, -1
@@ -497,17 +497,16 @@ define amdgpu_kernel void @test_div_fmas_f32_inline_imm_1(ptr addrspace(1) %out,
;
; GFX8-LABEL: test_div_fmas_f32_inline_imm_1:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x8
-; GFX8-NEXT: s_load_dword s1, s[4:5], 0x10
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x34
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: s_cmp_lg_u32 s2, 0
+; GFX8-NEXT: s_load_dword s3, s[4:5], 0x10
+; GFX8-NEXT: s_load_dword s4, s[4:5], 0x34
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: s_cmp_lg_u32 s4, 0
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
; GFX8-NEXT: v_div_fmas_f32 v2, v0, 1.0, v1
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -515,64 +514,60 @@ define amdgpu_kernel void @test_div_fmas_f32_inline_imm_1(ptr addrspace(1) %out,
;
; GFX10_W32-LABEL: test_div_fmas_f32_inline_imm_1:
; GFX10_W32: ; %bb.0:
-; GFX10_W32-NEXT: s_clause 0x3
-; GFX10_W32-NEXT: s_load_dword s2, s[4:5], 0x10
-; GFX10_W32-NEXT: s_load_dword s3, s[4:5], 0x34
-; GFX10_W32-NEXT: s_load_dword s6, s[4:5], 0x8
-; GFX10_W32-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10_W32-NEXT: s_clause 0x2
+; GFX10_W32-NEXT: s_load_dword s6, s[4:5], 0x10
+; GFX10_W32-NEXT: s_load_dword s7, s[4:5], 0x34
+; GFX10_W32-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10_W32-NEXT: v_mov_b32_e32 v1, 0
; GFX10_W32-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10_W32-NEXT: v_mov_b32_e32 v0, s2
-; GFX10_W32-NEXT: s_cmp_lg_u32 s3, 0
+; GFX10_W32-NEXT: v_mov_b32_e32 v0, s6
+; GFX10_W32-NEXT: s_cmp_lg_u32 s7, 0
; GFX10_W32-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX10_W32-NEXT: v_div_fmas_f32 v0, s6, 1.0, v0
+; GFX10_W32-NEXT: v_div_fmas_f32 v0, s2, 1.0, v0
; GFX10_W32-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10_W32-NEXT: s_endpgm
;
; GFX10_W64-LABEL: test_div_fmas_f32_inline_imm_1:
; GFX10_W64: ; %bb.0:
-; GFX10_W64-NEXT: s_clause 0x3
-; GFX10_W64-NEXT: s_load_dword s2, s[4:5], 0x10
-; GFX10_W64-NEXT: s_load_dword s3, s[4:5], 0x34
-; GFX10_W64-NEXT: s_load_dword s6, s[4:5], 0x8
-; GFX10_W64-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10_W64-NEXT: s_clause 0x2
+; GFX10_W64-NEXT: s_load_dword s6, s[4:5], 0x10
+; GFX10_W64-NEXT: s_load_dword s7, s[4:5], 0x34
+; GFX10_W64-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10_W64-NEXT: v_mov_b32_e32 v1, 0
; GFX10_W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10_W64-NEXT: v_mov_b32_e32 v0, s2
-; GFX10_W64-NEXT: s_cmp_lg_u32 s3, 0
+; GFX10_W64-NEXT: v_mov_b32_e32 v0, s6
+; GFX10_W64-NEXT: s_cmp_lg_u32 s7, 0
; GFX10_W64-NEXT: s_cselect_b64 vcc, exec, 0
-; GFX10_W64-NEXT: v_div_fmas_f32 v0, s6, 1.0, v0
+; GFX10_W64-NEXT: v_div_fmas_f32 v0, s2, 1.0, v0
; GFX10_W64-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10_W64-NEXT: s_endpgm
;
; GFX11_W32-LABEL: test_div_fmas_f32_inline_imm_1:
; GFX11_W32: ; %bb.0:
-; GFX11_W32-NEXT: s_clause 0x3
-; GFX11_W32-NEXT: s_load_b32 s2, s[4:5], 0x10
-; GFX11_W32-NEXT: s_load_b32 s3, s[4:5], 0x34
-; GFX11_W32-NEXT: s_load_b32 s6, s[4:5], 0x8
-; GFX11_W32-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11_W32-NEXT: s_clause 0x2
+; GFX11_W32-NEXT: s_load_b32 s6, s[4:5], 0x10
+; GFX11_W32-NEXT: s_load_b32 s7, s[4:5], 0x34
+; GFX11_W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11_W32-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11_W32-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
-; GFX11_W32-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11_W32-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s6
+; GFX11_W32-NEXT: s_cmp_lg_u32 s7, 0
; GFX11_W32-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11_W32-NEXT: v_div_fmas_f32 v0, s6, 1.0, v0
+; GFX11_W32-NEXT: v_div_fmas_f32 v0, s2, 1.0, v0
; GFX11_W32-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11_W32-NEXT: s_endpgm
;
; GFX11_W64-LABEL: test_div_fmas_f32_inline_imm_1:
; GFX11_W64: ; %bb.0:
-; GFX11_W64-NEXT: s_clause 0x3
-; GFX11_W64-NEXT: s_load_b32 s2, s[4:5], 0x10
-; GFX11_W64-NEXT: s_load_b32 s3, s[4:5], 0x34
-; GFX11_W64-NEXT: s_load_b32 s6, s[4:5], 0x8
-; GFX11_W64-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11_W64-NEXT: s_clause 0x2
+; GFX11_W64-NEXT: s_load_b32 s6, s[4:5], 0x10
+; GFX11_W64-NEXT: s_load_b32 s7, s[4:5], 0x34
+; GFX11_W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11_W64-NEXT: v_mov_b32_e32 v1, 0
; GFX11_W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11_W64-NEXT: v_mov_b32_e32 v0, s2
-; GFX11_W64-NEXT: s_cmp_lg_u32 s3, 0
+; GFX11_W64-NEXT: v_mov_b32_e32 v0, s6
+; GFX11_W64-NEXT: s_cmp_lg_u32 s7, 0
; GFX11_W64-NEXT: s_cselect_b64 vcc, exec, 0
-; GFX11_W64-NEXT: v_div_fmas_f32 v0, s6, 1.0, v0
+; GFX11_W64-NEXT: v_div_fmas_f32 v0, s2, 1.0, v0
; GFX11_W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11_W64-NEXT: s_endpgm
%result = call float @llvm.amdgcn.div.fmas.f32(float %a, float 1.0, float %c, i1 %d)
@@ -795,32 +790,31 @@ define amdgpu_kernel void @test_div_fmas_f64(ptr addrspace(1) %out, double %a, d
define amdgpu_kernel void @test_div_fmas_f32_cond_to_vcc(ptr addrspace(1) %out, float %a, float %b, float %c, i32 %i) {
; GFX7-LABEL: test_div_fmas_f32_cond_to_vcc:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
-; GFX7-NEXT: s_mov_b32 s6, -1
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x4
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_cmp_eq_u32 s3, 0
-; GFX7-NEXT: v_mov_b32_e32 v0, s0
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
+; GFX7-NEXT: s_cmp_eq_u32 s7, 0
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: v_mov_b32_e32 v2, s6
; GFX7-NEXT: s_cselect_b64 vcc, exec, 0
; GFX7-NEXT: v_div_fmas_f32 v0, v0, v1, v2
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: test_div_fmas_f32_cond_to_vcc:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x8
+; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_cmp_eq_u32 s3, 0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: s_cmp_eq_u32 s7, 0
+; GFX8-NEXT: v_mov_b32_e32 v0, s2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
; GFX8-NEXT: s_cselect_b64 vcc, exec, 0
; GFX8-NEXT: v_div_fmas_f32 v2, v0, v1, v2
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
@@ -829,60 +823,60 @@ define amdgpu_kernel void @test_div_fmas_f32_cond_to_vcc(ptr addrspace(1) %out,
; GFX10_W32-LABEL: test_div_fmas_f32_cond_to_vcc:
; GFX10_W32: ; %bb.0:
; GFX10_W32-NEXT: s_clause 0x1
-; GFX10_W32-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x8
-; GFX10_W32-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX10_W32-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX10_W32-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10_W32-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10_W32-NEXT: v_mov_b32_e32 v0, s1
-; GFX10_W32-NEXT: v_mov_b32_e32 v1, s2
-; GFX10_W32-NEXT: s_cmp_eq_u32 s3, 0
+; GFX10_W32-NEXT: v_mov_b32_e32 v1, s6
+; GFX10_W32-NEXT: v_mov_b32_e32 v0, s3
+; GFX10_W32-NEXT: s_cmp_eq_u32 s7, 0
; GFX10_W32-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX10_W32-NEXT: v_div_fmas_f32 v0, s0, v0, v1
+; GFX10_W32-NEXT: v_div_fmas_f32 v0, s2, v0, v1
; GFX10_W32-NEXT: v_mov_b32_e32 v1, 0
-; GFX10_W32-NEXT: global_store_dword v1, v0, s[6:7]
+; GFX10_W32-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10_W32-NEXT: s_endpgm
;
; GFX10_W64-LABEL: test_div_fmas_f32_cond_to_vcc:
; GFX10_W64: ; %bb.0:
; GFX10_W64-NEXT: s_clause 0x1
-; GFX10_W64-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x8
-; GFX10_W64-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; GFX10_W64-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX10_W64-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10_W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10_W64-NEXT: v_mov_b32_e32 v0, s1
-; GFX10_W64-NEXT: v_mov_b32_e32 v1, s2
-; GFX10_W64-NEXT: s_cmp_eq_u32 s3, 0
+; GFX10_W64-NEXT: v_mov_b32_e32 v1, s6
+; GFX10_W64-NEXT: v_mov_b32_e32 v0, s3
+; GFX10_W64-NEXT: s_cmp_eq_u32 s7, 0
; GFX10_W64-NEXT: s_cselect_b64 vcc, exec, 0
-; GFX10_W64-NEXT: v_div_fmas_f32 v0, s0, v0, v1
+; GFX10_W64-NEXT: v_div_fmas_f32 v0, s2, v0, v1
; GFX10_W64-NEXT: v_mov_b32_e32 v1, 0
-; GFX10_W64-NEXT: global_store_dword v1, v0, s[6:7]
+; GFX10_W64-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10_W64-NEXT: s_endpgm
;
; GFX11_W32-LABEL: test_div_fmas_f32_cond_to_vcc:
; GFX11_W32: ; %bb.0:
; GFX11_W32-NEXT: s_clause 0x1
-; GFX11_W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
-; GFX11_W32-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GFX11_W32-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX11_W32-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11_W32-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11_W32-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v1, s2
-; GFX11_W32-NEXT: s_cmp_eq_u32 s3, 0
+; GFX11_W32-NEXT: s_cmp_eq_u32 s7, 0
+; GFX11_W32-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s6
; GFX11_W32-NEXT: s_cselect_b32 vcc_lo, exec_lo, 0
-; GFX11_W32-NEXT: v_div_fmas_f32 v0, s0, v0, v1
+; GFX11_W32-NEXT: v_div_fmas_f32 v0, s2, v0, v1
; GFX11_W32-NEXT: v_mov_b32_e32 v1, 0
-; GFX11_W32-NEXT: global_store_b32 v1, v0, s[4:5]
+; GFX11_W32-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11_W32-NEXT: s_endpgm
;
; GFX11_W64-LABEL: test_div_fmas_f32_cond_to_vcc:
; GFX11_W64: ; %bb.0:
; GFX11_W64-NEXT: s_clause 0x1
-; GFX11_W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
-; GFX11_W64-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GFX11_W64-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX11_W64-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11_W64-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11_W64-NEXT: v_mov_b32_e32 v0, s1
-; GFX11_W64-NEXT: v_mov_b32_e32 v1, s2
-; GFX11_W64-NEXT: s_cmp_eq_u32 s3, 0
+; GFX11_W64-NEXT: v_mov_b32_e32 v1, s6
+; GFX11_W64-NEXT: v_mov_b32_e32 v0, s3
+; GFX11_W64-NEXT: s_cmp_eq_u32 s7, 0
; GFX11_W64-NEXT: s_cselect_b64 vcc, exec, 0
-; GFX11_W64-NEXT: v_div_fmas_f32 v0, s0, v0, v1
+; GFX11_W64-NEXT: v_div_fmas_f32 v0, s2, v0, v1
; GFX11_W64-NEXT: v_mov_b32_e32 v1, 0
-; GFX11_W64-NEXT: global_store_b32 v1, v0, s[4:5]
+; GFX11_W64-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11_W64-NEXT: s_endpgm
%cmp = icmp eq i32 %i, 0
%result = call float @llvm.amdgcn.div.fmas.f32(float %a, float %b, float %c, i1 %cmp)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd-with-ret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd-with-ret.ll
index 2046459a406a4..54f5e4c239d24 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd-with-ret.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.raw.ptr.buffer.atomic.fadd-with-ret.ll
@@ -4,7 +4,7 @@
declare float @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.f32(float, ptr addrspace(8), i32, i32, i32 immarg)
declare <2 x half> @llvm.amdgcn.raw.ptr.buffer.atomic.fadd.v2f16(<2 x half>, ptr addrspace(8), i32, i32, i32 immarg)
-; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, 0 :: (volatile dereferenceable load store (s32) on %ir.rsrc.load, align 1, addrspace 8) (in function: buffer_atomic_add_f32_rtn)
+; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, 0 :: (volatile dereferenceable load store (s32) on {{%ir\.[^,)]+}}, align 1, addrspace 8) (in function: buffer_atomic_add_f32_rtn)
; GFX90A-LABEL: {{^}}buffer_atomic_add_f32_rtn:
; GFX90A: buffer_atomic_add_f32 v{{[0-9]+}}, v{{[0-9]+}}, s[{{[0-9:]+}}], s{{[0-9]+}} offen glc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll
index 3571bf191fcd3..aef1c14657da6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.set.inactive.ll
@@ -4,17 +4,18 @@
define amdgpu_kernel void @set_inactive(ptr addrspace(1) %out, i32 %in) {
; GCN-LABEL: set_inactive:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 42, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 42, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %in, i32 42) #0
%tmp = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %tmp.0)
@@ -88,17 +89,15 @@ define amdgpu_kernel void @set_inactive_imm_poison_64(ptr addrspace(1) %out) {
define amdgpu_kernel void @set_inactive_scc(ptr addrspace(1) %out, i32 %in, <4 x i32> inreg %desc) {
; GCN-LABEL: set_inactive_scc:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_buffer_load_dword s7, s[0:3], 0x0
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: v_mov_b32_e32 v1, s6
+; GCN-NEXT: s_buffer_load_dword s4, s[4:7], 0x0
+; GCN-NEXT: v_mov_b32_e32 v1, s2
; GCN-NEXT: s_or_saveexec_b64 s[2:3], -1
; GCN-NEXT: v_cndmask_b32_e64 v0, 42, v1, s[2:3]
; GCN-NEXT: s_mov_b64 exec, s[2:3]
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_lg_u32 s7, 56
+; GCN-NEXT: s_cmp_lg_u32 s4, 56
; GCN-NEXT: v_mov_b32_e32 v1, v0
; GCN-NEXT: s_mov_b64 s[2:3], -1
; GCN-NEXT: s_cbranch_scc1 .LBB4_3
@@ -140,18 +139,19 @@ define amdgpu_kernel void @set_inactive_scc(ptr addrspace(1) %out, i32 %in, <4 x
define amdgpu_kernel void @set_inactive_f32(ptr addrspace(1) %out, float %in) {
; GCN-LABEL: set_inactive_f32:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
; GCN-NEXT: v_mov_b32_e32 v0, 0x40400000
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call float @llvm.amdgcn.set.inactive.f32(float %in, float 3.0) #0
%tmp = call float @llvm.amdgcn.strict.wwm.f32(float %tmp.0)
@@ -191,18 +191,19 @@ define amdgpu_kernel void @set_inactive_f64(ptr addrspace(1) %out, double %in) {
define amdgpu_kernel void @set_inactive_v2i16(ptr addrspace(1) %out, <2 x i16> %in) {
; GCN-LABEL: set_inactive_v2i16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
; GCN-NEXT: v_mov_b32_e32 v0, 0x10001
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call <2 x i16> @llvm.amdgcn.set.inactive.v2i16(<2 x i16> %in, <2 x i16> <i16 1, i16 1>) #0
%tmp = call <2 x i16> @llvm.amdgcn.strict.wwm.v2i16(<2 x i16> %tmp.0)
@@ -213,18 +214,19 @@ define amdgpu_kernel void @set_inactive_v2i16(ptr addrspace(1) %out, <2 x i16> %
define amdgpu_kernel void @set_inactive_v2f16(ptr addrspace(1) %out, <2 x half> %in) {
; GCN-LABEL: set_inactive_v2f16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
; GCN-NEXT: v_mov_b32_e32 v0, 0x3c003c00
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call <2 x half> @llvm.amdgcn.set.inactive.v2f16(<2 x half> %in, <2 x half> <half 1.0, half 1.0>) #0
%tmp = call <2 x half> @llvm.amdgcn.strict.wwm.v2i16(<2 x half> %tmp.0)
@@ -289,18 +291,19 @@ define amdgpu_kernel void @set_inactive_v2f32(ptr addrspace(1) %out, <2 x float>
define amdgpu_kernel void @set_inactive_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) {
; GCN-LABEL: set_inactive_v2bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
; GCN-NEXT: v_mov_b32_e32 v0, 0x3f803f80
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call <2 x bfloat> @llvm.amdgcn.set.inactive.v2bf16(<2 x bfloat> %in, <2 x bfloat> <bfloat 1.0, bfloat 1.0>) #0
%tmp = call <2 x bfloat> @llvm.amdgcn.strict.wwm.v2bf16(<2 x bfloat> %tmp.0)
@@ -422,17 +425,18 @@ define amdgpu_kernel void @set_inactive_p0(ptr addrspace(1) %out, ptr %in) {
define amdgpu_kernel void @set_inactive_p2(ptr addrspace(1) %out, ptr addrspace(2) %in) {
; GCN-LABEL: set_inactive_p2:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call ptr addrspace(2) @llvm.amdgcn.set.inactive.p2(ptr addrspace(2) %in, ptr addrspace(2) zeroinitializer) #0
%tmp = call ptr addrspace(2) @llvm.amdgcn.strict.wwm.p2(ptr addrspace(2) %tmp.0)
@@ -443,17 +447,18 @@ define amdgpu_kernel void @set_inactive_p2(ptr addrspace(1) %out, ptr addrspace(
define amdgpu_kernel void @set_inactive_p3(ptr addrspace(1) %out, ptr addrspace(3) %in) {
; GCN-LABEL: set_inactive_p3:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call ptr addrspace(3) @llvm.amdgcn.set.inactive.p3(ptr addrspace(3) %in, ptr addrspace(3) zeroinitializer) #0
%tmp = call ptr addrspace(3) @llvm.amdgcn.strict.wwm.p3(ptr addrspace(3) %tmp.0)
@@ -464,17 +469,18 @@ define amdgpu_kernel void @set_inactive_p3(ptr addrspace(1) %out, ptr addrspace(
define amdgpu_kernel void @set_inactive_p5(ptr addrspace(1) %out, ptr addrspace(5) %in) {
; GCN-LABEL: set_inactive_p5:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call ptr addrspace(5) @llvm.amdgcn.set.inactive.p5(ptr addrspace(5) %in, ptr addrspace(5) zeroinitializer) #0
%tmp = call ptr addrspace(5) @llvm.amdgcn.strict.wwm.p5(ptr addrspace(5) %tmp.0)
@@ -485,17 +491,18 @@ define amdgpu_kernel void @set_inactive_p5(ptr addrspace(1) %out, ptr addrspace(
define amdgpu_kernel void @set_inactive_p6(ptr addrspace(1) %out, ptr addrspace(6) %in) {
; GCN-LABEL: set_inactive_p6:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call ptr addrspace(6) @llvm.amdgcn.set.inactive.p6(ptr addrspace(6) %in, ptr addrspace(6) null) #0
%tmp = call ptr addrspace(6) @llvm.amdgcn.strict.wwm.p6(ptr addrspace(6) %tmp.0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll
index bb07edcea5900..c70b51fbe4808 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.struct.ptr.buffer.atomic.fadd-with-ret.ll
@@ -1,7 +1,7 @@
; RUN: llc -global-isel -new-reg-bank-select -mtriple=amdgcn-amd-amdhsa -mcpu=gfx90a < %s | FileCheck -enable-var-scope -check-prefixes=GFX90A %s
; RUN: not llc -global-isel -new-reg-bank-select < %s -mtriple=amdgcn -mcpu=gfx908 2>&1 | FileCheck %s -check-prefix=GFX908
-; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, -1 :: (volatile dereferenceable load store (s32) on %ir.rsrc.load, align 1, addrspace 8) (in function: buffer_atomic_add_f32_rtn)
+; GFX908: LLVM ERROR: cannot select: %{{[0-9]+}}:vgpr_32(s32) = G_AMDGPU_BUFFER_ATOMIC_FADD %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr(<4 x s32>), %{{[0-9]+}}:vgpr(s32), %{{[0-9]+}}:vgpr, %{{[0-9]+}}:sgpr, 0, 0, -1 :: (volatile dereferenceable load store (s32) on {{%ir\.[^,)]+}}, align 1, addrspace 8) (in function: buffer_atomic_add_f32_rtn)
declare float @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.f32(float, ptr addrspace(8), i32, i32, i32, i32 immarg)
declare <2 x half> @llvm.amdgcn.struct.ptr.buffer.atomic.fadd.v2f16(<2 x half>, ptr addrspace(8), i32, i32, i32, i32 immarg)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll
index b58e04d80a229..e50607d93e11b 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.trig.preop.ll
@@ -40,8 +40,7 @@ define double @v_trig_preop_f64_imm(double %a, i32 %b) {
define amdgpu_kernel void @s_trig_preop_f64(double %a, i32 %b) {
; CI-LABEL: s_trig_preop_f64:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
@@ -60,8 +59,7 @@ define amdgpu_kernel void @s_trig_preop_f64(double %a, i32 %b) {
;
; VI-LABEL: s_trig_preop_f64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
@@ -80,8 +78,7 @@ define amdgpu_kernel void @s_trig_preop_f64(double %a, i32 %b) {
;
; GFX9-LABEL: s_trig_preop_f64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: s_add_u32 flat_scratch_lo, s12, s17
; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -97,9 +94,7 @@ define amdgpu_kernel void @s_trig_preop_f64(double %a, i32 %b) {
; GFX10-NEXT: s_addc_u32 s13, s13, 0
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
; GFX10-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_trig_preop_f64 v[0:1], s[0:1], s2
; GFX10-NEXT: flat_store_dwordx2 v[0:1], v[0:1]
@@ -108,9 +103,7 @@ define amdgpu_kernel void @s_trig_preop_f64(double %a, i32 %b) {
;
; GFX11-LABEL: s_trig_preop_f64:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_trig_preop_f64 v[0:1], s[0:1], s2
; GFX11-NEXT: flat_store_b64 v[0:1], v[0:1] dlc
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
index cb01b93dc8403..54e119c218194 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/sdivrem.ll
@@ -1018,546 +1018,546 @@ define amdgpu_kernel void @sdivrem_v2i32(ptr addrspace(1) %out0, ptr addrspace(1
define amdgpu_kernel void @sdivrem_v4i32(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <4 x i32> %x, <4 x i32> %y) {
; GFX8-LABEL: sdivrem_v4i32:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
; GFX8-NEXT: s_add_i32 s12, s12, s17
-; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT: s_load_dwordx4 s[12:15], s[8:9], 0x20
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_ashr_i32 s10, s4, 31
-; GFX8-NEXT: s_add_i32 s4, s4, s10
-; GFX8-NEXT: s_xor_b32 s4, s4, s10
-; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s4
-; GFX8-NEXT: s_sub_i32 s11, 0, s4
+; GFX8-NEXT: s_ashr_i32 s10, s12, 31
+; GFX8-NEXT: s_add_i32 s0, s12, s10
+; GFX8-NEXT: s_xor_b32 s11, s0, s10
+; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s11
+; GFX8-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX8-NEXT: s_sub_i32 s8, 0, s11
; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v11, s3
+; GFX8-NEXT: v_mov_b32_e32 v10, s2
; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT: v_mov_b32_e32 v9, s1
+; GFX8-NEXT: v_mov_b32_e32 v8, s0
+; GFX8-NEXT: v_readfirstlane_b32 s9, v0
+; GFX8-NEXT: s_mul_i32 s8, s8, s9
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s8
+; GFX8-NEXT: s_ashr_i32 s8, s4, 31
+; GFX8-NEXT: s_add_i32 s4, s4, s8
+; GFX8-NEXT: s_xor_b32 s4, s4, s8
; GFX8-NEXT: v_readfirstlane_b32 s12, v0
-; GFX8-NEXT: s_mul_i32 s11, s11, s12
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s11
-; GFX8-NEXT: s_ashr_i32 s11, s0, 31
-; GFX8-NEXT: s_add_i32 s0, s0, s11
-; GFX8-NEXT: s_xor_b32 s0, s0, s11
-; GFX8-NEXT: v_readfirstlane_b32 s13, v0
-; GFX8-NEXT: s_add_i32 s12, s12, s13
-; GFX8-NEXT: v_mov_b32_e32 v0, s12
-; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0
-; GFX8-NEXT: v_readfirstlane_b32 s12, v0
-; GFX8-NEXT: s_mul_i32 s13, s12, s4
-; GFX8-NEXT: s_sub_i32 s0, s0, s13
-; GFX8-NEXT: s_cmp_ge_u32 s0, s4
-; GFX8-NEXT: s_cselect_b32 s13, 1, 0
-; GFX8-NEXT: s_add_i32 s14, s12, 1
-; GFX8-NEXT: s_cmp_lg_u32 s13, 0
-; GFX8-NEXT: s_cselect_b32 s12, s14, s12
-; GFX8-NEXT: s_sub_i32 s14, s0, s4
-; GFX8-NEXT: s_cmp_lg_u32 s13, 0
-; GFX8-NEXT: s_cselect_b32 s0, s14, s0
-; GFX8-NEXT: s_cmp_ge_u32 s0, s4
-; GFX8-NEXT: s_cselect_b32 s13, 1, 0
-; GFX8-NEXT: s_add_i32 s14, s12, 1
-; GFX8-NEXT: s_cmp_lg_u32 s13, 0
-; GFX8-NEXT: s_cselect_b32 s12, s14, s12
-; GFX8-NEXT: s_sub_i32 s4, s0, s4
-; GFX8-NEXT: s_cmp_lg_u32 s13, 0
-; GFX8-NEXT: s_cselect_b32 s0, s4, s0
-; GFX8-NEXT: s_ashr_i32 s13, s5, 31
-; GFX8-NEXT: s_add_i32 s4, s5, s13
-; GFX8-NEXT: s_xor_b32 s5, s4, s13
-; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX8-NEXT: s_sub_i32 s14, 0, s5
-; GFX8-NEXT: s_xor_b32 s4, s11, s10
-; GFX8-NEXT: s_ashr_i32 s10, s1, 31
-; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX8-NEXT: s_add_i32 s1, s1, s10
-; GFX8-NEXT: s_xor_b32 s1, s1, s10
-; GFX8-NEXT: s_xor_b32 s12, s12, s4
-; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX8-NEXT: v_readfirstlane_b32 s15, v0
-; GFX8-NEXT: s_mul_i32 s14, s14, s15
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s14
-; GFX8-NEXT: v_readfirstlane_b32 s14, v0
-; GFX8-NEXT: s_add_i32 s14, s15, s14
-; GFX8-NEXT: v_mov_b32_e32 v0, s14
-; GFX8-NEXT: v_mul_hi_u32 v0, s1, v0
-; GFX8-NEXT: s_xor_b32 s14, s0, s11
-; GFX8-NEXT: s_sub_i32 s0, s12, s4
-; GFX8-NEXT: s_sub_i32 s4, s14, s11
-; GFX8-NEXT: v_readfirstlane_b32 s11, v0
-; GFX8-NEXT: s_mul_i32 s12, s11, s5
-; GFX8-NEXT: s_sub_i32 s1, s1, s12
-; GFX8-NEXT: s_cmp_ge_u32 s1, s5
+; GFX8-NEXT: s_add_i32 s9, s9, s12
+; GFX8-NEXT: v_mov_b32_e32 v0, s9
+; GFX8-NEXT: v_mul_hi_u32 v0, s4, v0
+; GFX8-NEXT: v_readfirstlane_b32 s9, v0
+; GFX8-NEXT: s_mul_i32 s12, s9, s11
+; GFX8-NEXT: s_sub_i32 s4, s4, s12
+; GFX8-NEXT: s_cmp_ge_u32 s4, s11
; GFX8-NEXT: s_cselect_b32 s12, 1, 0
-; GFX8-NEXT: s_add_i32 s14, s11, 1
+; GFX8-NEXT: s_add_i32 s16, s9, 1
; GFX8-NEXT: s_cmp_lg_u32 s12, 0
-; GFX8-NEXT: s_cselect_b32 s11, s14, s11
-; GFX8-NEXT: s_sub_i32 s14, s1, s5
+; GFX8-NEXT: s_cselect_b32 s9, s16, s9
+; GFX8-NEXT: s_sub_i32 s16, s4, s11
; GFX8-NEXT: s_cmp_lg_u32 s12, 0
-; GFX8-NEXT: s_cselect_b32 s1, s14, s1
-; GFX8-NEXT: s_cmp_ge_u32 s1, s5
+; GFX8-NEXT: s_cselect_b32 s4, s16, s4
+; GFX8-NEXT: s_cmp_ge_u32 s4, s11
; GFX8-NEXT: s_cselect_b32 s12, 1, 0
-; GFX8-NEXT: s_add_i32 s14, s11, 1
+; GFX8-NEXT: s_add_i32 s16, s9, 1
; GFX8-NEXT: s_cmp_lg_u32 s12, 0
-; GFX8-NEXT: s_cselect_b32 s11, s14, s11
-; GFX8-NEXT: s_sub_i32 s5, s1, s5
+; GFX8-NEXT: s_cselect_b32 s9, s16, s9
+; GFX8-NEXT: s_sub_i32 s11, s4, s11
; GFX8-NEXT: s_cmp_lg_u32 s12, 0
-; GFX8-NEXT: s_cselect_b32 s1, s5, s1
-; GFX8-NEXT: s_ashr_i32 s12, s6, 31
-; GFX8-NEXT: s_add_i32 s5, s6, s12
-; GFX8-NEXT: s_xor_b32 s6, s5, s12
-; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX8-NEXT: s_sub_i32 s14, 0, s6
-; GFX8-NEXT: s_xor_b32 s5, s10, s13
-; GFX8-NEXT: s_ashr_i32 s13, s2, 31
-; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX8-NEXT: s_add_i32 s2, s2, s13
-; GFX8-NEXT: s_xor_b32 s2, s2, s13
-; GFX8-NEXT: s_xor_b32 s11, s11, s5
-; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX8-NEXT: v_readfirstlane_b32 s15, v0
-; GFX8-NEXT: s_mul_i32 s14, s14, s15
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s14
-; GFX8-NEXT: v_readfirstlane_b32 s14, v0
-; GFX8-NEXT: s_add_i32 s14, s15, s14
-; GFX8-NEXT: v_mov_b32_e32 v0, s14
-; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX8-NEXT: s_xor_b32 s14, s1, s10
-; GFX8-NEXT: s_sub_i32 s1, s11, s5
-; GFX8-NEXT: s_sub_i32 s5, s14, s10
-; GFX8-NEXT: v_readfirstlane_b32 s10, v0
-; GFX8-NEXT: s_mul_i32 s11, s10, s6
-; GFX8-NEXT: s_sub_i32 s2, s2, s11
-; GFX8-NEXT: s_cmp_ge_u32 s2, s6
-; GFX8-NEXT: s_cselect_b32 s11, 1, 0
-; GFX8-NEXT: s_add_i32 s14, s10, 1
-; GFX8-NEXT: s_cmp_lg_u32 s11, 0
-; GFX8-NEXT: s_cselect_b32 s10, s14, s10
-; GFX8-NEXT: s_sub_i32 s14, s2, s6
-; GFX8-NEXT: s_cmp_lg_u32 s11, 0
-; GFX8-NEXT: s_cselect_b32 s2, s14, s2
-; GFX8-NEXT: s_cmp_ge_u32 s2, s6
-; GFX8-NEXT: s_cselect_b32 s11, 1, 0
-; GFX8-NEXT: s_add_i32 s14, s10, 1
-; GFX8-NEXT: s_cmp_lg_u32 s11, 0
-; GFX8-NEXT: s_cselect_b32 s14, s14, s10
-; GFX8-NEXT: s_sub_i32 s6, s2, s6
-; GFX8-NEXT: s_cmp_lg_u32 s11, 0
-; GFX8-NEXT: s_cselect_b32 s2, s6, s2
-; GFX8-NEXT: s_ashr_i32 s15, s7, 31
-; GFX8-NEXT: s_add_i32 s6, s7, s15
-; GFX8-NEXT: s_xor_b32 s7, s6, s15
-; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX8-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX8-NEXT: s_xor_b32 s6, s13, s12
+; GFX8-NEXT: s_cselect_b32 s4, s11, s4
+; GFX8-NEXT: s_ashr_i32 s11, s13, 31
+; GFX8-NEXT: s_add_i32 s12, s13, s11
+; GFX8-NEXT: s_xor_b32 s12, s12, s11
+; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s12
+; GFX8-NEXT: s_sub_i32 s16, 0, s12
+; GFX8-NEXT: s_ashr_i32 s13, s5, 31
+; GFX8-NEXT: s_add_i32 s5, s5, s13
; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v8, s8
-; GFX8-NEXT: v_mov_b32_e32 v10, s10
+; GFX8-NEXT: s_xor_b32 s5, s5, s13
+; GFX8-NEXT: s_xor_b32 s10, s8, s10
+; GFX8-NEXT: s_xor_b32 s9, s9, s10
; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX8-NEXT: v_mov_b32_e32 v9, s9
-; GFX8-NEXT: v_mov_b32_e32 v11, s11
-; GFX8-NEXT: s_sub_i32 s9, 0, s7
-; GFX8-NEXT: v_readfirstlane_b32 s10, v0
-; GFX8-NEXT: s_mul_i32 s9, s9, s10
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s9
-; GFX8-NEXT: s_ashr_i32 s8, s3, 31
-; GFX8-NEXT: s_add_i32 s3, s3, s8
-; GFX8-NEXT: s_xor_b32 s3, s3, s8
-; GFX8-NEXT: v_readfirstlane_b32 s11, v0
-; GFX8-NEXT: s_add_i32 s10, s10, s11
-; GFX8-NEXT: v_mov_b32_e32 v0, s10
-; GFX8-NEXT: v_mul_hi_u32 v0, s3, v0
-; GFX8-NEXT: s_xor_b32 s9, s14, s6
-; GFX8-NEXT: s_xor_b32 s10, s2, s13
-; GFX8-NEXT: s_sub_i32 s2, s9, s6
+; GFX8-NEXT: v_readfirstlane_b32 s17, v0
+; GFX8-NEXT: s_mul_i32 s16, s16, s17
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s16
+; GFX8-NEXT: v_readfirstlane_b32 s16, v0
+; GFX8-NEXT: s_add_i32 s16, s17, s16
+; GFX8-NEXT: v_mov_b32_e32 v0, s16
+; GFX8-NEXT: v_mul_hi_u32 v0, s5, v0
+; GFX8-NEXT: s_xor_b32 s16, s4, s8
+; GFX8-NEXT: s_sub_i32 s4, s9, s10
+; GFX8-NEXT: s_sub_i32 s8, s16, s8
; GFX8-NEXT: v_readfirstlane_b32 s9, v0
-; GFX8-NEXT: s_sub_i32 s6, s10, s13
-; GFX8-NEXT: s_mul_i32 s10, s9, s7
-; GFX8-NEXT: s_sub_i32 s3, s3, s10
-; GFX8-NEXT: s_cmp_ge_u32 s3, s7
+; GFX8-NEXT: s_mul_i32 s10, s9, s12
+; GFX8-NEXT: s_sub_i32 s5, s5, s10
+; GFX8-NEXT: s_cmp_ge_u32 s5, s12
; GFX8-NEXT: s_cselect_b32 s10, 1, 0
-; GFX8-NEXT: s_add_i32 s11, s9, 1
+; GFX8-NEXT: s_add_i32 s16, s9, 1
; GFX8-NEXT: s_cmp_lg_u32 s10, 0
-; GFX8-NEXT: s_cselect_b32 s9, s11, s9
-; GFX8-NEXT: s_sub_i32 s11, s3, s7
+; GFX8-NEXT: s_cselect_b32 s9, s16, s9
+; GFX8-NEXT: s_sub_i32 s16, s5, s12
; GFX8-NEXT: s_cmp_lg_u32 s10, 0
-; GFX8-NEXT: s_cselect_b32 s3, s11, s3
-; GFX8-NEXT: s_cmp_ge_u32 s3, s7
+; GFX8-NEXT: s_cselect_b32 s5, s16, s5
+; GFX8-NEXT: s_cmp_ge_u32 s5, s12
; GFX8-NEXT: s_cselect_b32 s10, 1, 0
-; GFX8-NEXT: s_add_i32 s11, s9, 1
+; GFX8-NEXT: s_add_i32 s16, s9, 1
; GFX8-NEXT: s_cmp_lg_u32 s10, 0
-; GFX8-NEXT: s_cselect_b32 s9, s11, s9
-; GFX8-NEXT: s_sub_i32 s7, s3, s7
+; GFX8-NEXT: s_cselect_b32 s9, s16, s9
+; GFX8-NEXT: s_sub_i32 s12, s5, s12
; GFX8-NEXT: s_cmp_lg_u32 s10, 0
-; GFX8-NEXT: s_cselect_b32 s3, s7, s3
-; GFX8-NEXT: s_xor_b32 s7, s8, s15
-; GFX8-NEXT: s_xor_b32 s9, s9, s7
-; GFX8-NEXT: s_xor_b32 s10, s3, s8
-; GFX8-NEXT: s_sub_i32 s3, s9, s7
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: s_sub_i32 s7, s10, s8
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_mov_b32_e32 v4, s4
-; GFX8-NEXT: v_mov_b32_e32 v5, s5
-; GFX8-NEXT: v_mov_b32_e32 v6, s6
-; GFX8-NEXT: v_mov_b32_e32 v7, s7
+; GFX8-NEXT: s_cselect_b32 s5, s12, s5
+; GFX8-NEXT: s_ashr_i32 s10, s14, 31
+; GFX8-NEXT: s_add_i32 s12, s14, s10
+; GFX8-NEXT: s_xor_b32 s12, s12, s10
+; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s12
+; GFX8-NEXT: s_sub_i32 s16, 0, s12
+; GFX8-NEXT: s_ashr_i32 s14, s6, 31
+; GFX8-NEXT: s_add_i32 s6, s6, s14
+; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT: s_xor_b32 s6, s6, s14
+; GFX8-NEXT: s_xor_b32 s11, s13, s11
+; GFX8-NEXT: s_xor_b32 s9, s9, s11
+; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT: v_readfirstlane_b32 s17, v0
+; GFX8-NEXT: s_mul_i32 s16, s16, s17
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s16
+; GFX8-NEXT: v_readfirstlane_b32 s16, v0
+; GFX8-NEXT: s_add_i32 s16, s17, s16
+; GFX8-NEXT: v_mov_b32_e32 v0, s16
+; GFX8-NEXT: v_mul_hi_u32 v0, s6, v0
+; GFX8-NEXT: s_xor_b32 s16, s5, s13
+; GFX8-NEXT: s_sub_i32 s5, s9, s11
+; GFX8-NEXT: s_sub_i32 s9, s16, s13
+; GFX8-NEXT: v_readfirstlane_b32 s11, v0
+; GFX8-NEXT: s_mul_i32 s13, s11, s12
+; GFX8-NEXT: s_sub_i32 s6, s6, s13
+; GFX8-NEXT: s_cmp_ge_u32 s6, s12
+; GFX8-NEXT: s_cselect_b32 s13, 1, 0
+; GFX8-NEXT: s_add_i32 s16, s11, 1
+; GFX8-NEXT: s_cmp_lg_u32 s13, 0
+; GFX8-NEXT: s_cselect_b32 s11, s16, s11
+; GFX8-NEXT: s_sub_i32 s16, s6, s12
+; GFX8-NEXT: s_cmp_lg_u32 s13, 0
+; GFX8-NEXT: s_cselect_b32 s6, s16, s6
+; GFX8-NEXT: s_cmp_ge_u32 s6, s12
+; GFX8-NEXT: s_cselect_b32 s13, 1, 0
+; GFX8-NEXT: s_add_i32 s16, s11, 1
+; GFX8-NEXT: s_cmp_lg_u32 s13, 0
+; GFX8-NEXT: s_cselect_b32 s11, s16, s11
+; GFX8-NEXT: s_sub_i32 s12, s6, s12
+; GFX8-NEXT: s_cmp_lg_u32 s13, 0
+; GFX8-NEXT: s_cselect_b32 s6, s12, s6
+; GFX8-NEXT: s_ashr_i32 s12, s15, 31
+; GFX8-NEXT: s_add_i32 s13, s15, s12
+; GFX8-NEXT: s_xor_b32 s13, s13, s12
+; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s13
+; GFX8-NEXT: s_sub_i32 s2, 0, s13
+; GFX8-NEXT: s_xor_b32 s0, s14, s10
+; GFX8-NEXT: s_ashr_i32 s1, s7, 31
+; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT: s_add_i32 s7, s7, s1
+; GFX8-NEXT: s_xor_b32 s7, s7, s1
+; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX8-NEXT: v_readfirstlane_b32 s3, v0
+; GFX8-NEXT: s_mul_i32 s2, s2, s3
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s2
+; GFX8-NEXT: s_xor_b32 s2, s11, s0
+; GFX8-NEXT: v_readfirstlane_b32 s10, v0
+; GFX8-NEXT: s_add_i32 s3, s3, s10
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_mul_hi_u32 v0, s7, v0
+; GFX8-NEXT: s_xor_b32 s3, s6, s14
+; GFX8-NEXT: s_sub_i32 s6, s2, s0
+; GFX8-NEXT: s_sub_i32 s10, s3, s14
+; GFX8-NEXT: v_readfirstlane_b32 s0, v0
+; GFX8-NEXT: s_mul_i32 s2, s0, s13
+; GFX8-NEXT: s_sub_i32 s2, s7, s2
+; GFX8-NEXT: s_cmp_ge_u32 s2, s13
+; GFX8-NEXT: s_cselect_b32 s3, 1, 0
+; GFX8-NEXT: s_add_i32 s7, s0, 1
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s0, s7, s0
+; GFX8-NEXT: s_sub_i32 s7, s2, s13
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s2, s7, s2
+; GFX8-NEXT: s_cmp_ge_u32 s2, s13
+; GFX8-NEXT: s_cselect_b32 s3, 1, 0
+; GFX8-NEXT: s_add_i32 s7, s0, 1
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s0, s7, s0
+; GFX8-NEXT: s_sub_i32 s7, s2, s13
+; GFX8-NEXT: s_cmp_lg_u32 s3, 0
+; GFX8-NEXT: s_cselect_b32 s2, s7, s2
+; GFX8-NEXT: s_xor_b32 s3, s1, s12
+; GFX8-NEXT: s_xor_b32 s0, s0, s3
+; GFX8-NEXT: s_xor_b32 s2, s2, s1
+; GFX8-NEXT: s_sub_i32 s7, s0, s3
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: s_sub_i32 s11, s2, s1
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_mov_b32_e32 v4, s8
+; GFX8-NEXT: v_mov_b32_e32 v5, s9
+; GFX8-NEXT: v_mov_b32_e32 v6, s10
+; GFX8-NEXT: v_mov_b32_e32 v7, s11
; GFX8-NEXT: flat_store_dwordx4 v[8:9], v[0:3]
; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[4:7]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: sdivrem_v4i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
+; GFX9-NEXT: s_load_dwordx4 s[12:15], s[8:9], 0x20
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s10, s4, 31
-; GFX9-NEXT: s_add_i32 s4, s4, s10
-; GFX9-NEXT: s_xor_b32 s4, s4, s10
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4
-; GFX9-NEXT: s_sub_i32 s12, 0, s4
-; GFX9-NEXT: s_ashr_i32 s11, s0, 31
-; GFX9-NEXT: s_add_i32 s0, s0, s11
-; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s11
-; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_mul_i32 s12, s12, s13
-; GFX9-NEXT: s_mul_hi_u32 s12, s13, s12
-; GFX9-NEXT: s_add_i32 s13, s13, s12
-; GFX9-NEXT: s_mul_hi_u32 s12, s0, s13
-; GFX9-NEXT: s_mul_i32 s13, s12, s4
-; GFX9-NEXT: s_sub_i32 s0, s0, s13
-; GFX9-NEXT: s_cmp_ge_u32 s0, s4
-; GFX9-NEXT: s_cselect_b32 s13, 1, 0
-; GFX9-NEXT: s_add_i32 s14, s12, 1
-; GFX9-NEXT: s_cmp_lg_u32 s13, 0
-; GFX9-NEXT: s_cselect_b32 s12, s14, s12
-; GFX9-NEXT: s_sub_i32 s14, s0, s4
-; GFX9-NEXT: s_cmp_lg_u32 s13, 0
-; GFX9-NEXT: s_cselect_b32 s0, s14, s0
-; GFX9-NEXT: s_cmp_ge_u32 s0, s4
-; GFX9-NEXT: s_cselect_b32 s13, 1, 0
-; GFX9-NEXT: s_add_i32 s14, s12, 1
-; GFX9-NEXT: s_cmp_lg_u32 s13, 0
-; GFX9-NEXT: s_cselect_b32 s12, s14, s12
-; GFX9-NEXT: s_sub_i32 s4, s0, s4
-; GFX9-NEXT: s_cmp_lg_u32 s13, 0
-; GFX9-NEXT: s_cselect_b32 s0, s4, s0
-; GFX9-NEXT: s_ashr_i32 s13, s5, 31
-; GFX9-NEXT: s_add_i32 s4, s5, s13
-; GFX9-NEXT: s_xor_b32 s5, s4, s13
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX9-NEXT: s_xor_b32 s4, s11, s10
-; GFX9-NEXT: s_xor_b32 s12, s12, s4
-; GFX9-NEXT: s_xor_b32 s14, s0, s11
+; GFX9-NEXT: s_ashr_i32 s10, s12, 31
+; GFX9-NEXT: s_add_i32 s0, s12, s10
+; GFX9-NEXT: s_xor_b32 s11, s0, s10
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s11
+; GFX9-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX9-NEXT: s_sub_i32 s9, 0, s11
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT: s_sub_i32 s0, s12, s4
-; GFX9-NEXT: s_sub_i32 s4, s14, s11
-; GFX9-NEXT: s_sub_i32 s11, 0, s5
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_ashr_i32 s8, s4, 31
+; GFX9-NEXT: s_add_i32 s4, s4, s8
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: s_ashr_i32 s10, s1, 31
-; GFX9-NEXT: s_add_i32 s1, s1, s10
-; GFX9-NEXT: s_xor_b32 s1, s1, s10
+; GFX9-NEXT: s_xor_b32 s4, s4, s8
; GFX9-NEXT: v_readfirstlane_b32 s12, v0
-; GFX9-NEXT: s_mul_i32 s11, s11, s12
-; GFX9-NEXT: s_mul_hi_u32 s11, s12, s11
-; GFX9-NEXT: s_add_i32 s12, s12, s11
-; GFX9-NEXT: s_mul_hi_u32 s11, s1, s12
-; GFX9-NEXT: s_mul_i32 s12, s11, s5
-; GFX9-NEXT: s_sub_i32 s1, s1, s12
-; GFX9-NEXT: s_cmp_ge_u32 s1, s5
+; GFX9-NEXT: s_mul_i32 s9, s9, s12
+; GFX9-NEXT: s_mul_hi_u32 s9, s12, s9
+; GFX9-NEXT: s_add_i32 s12, s12, s9
+; GFX9-NEXT: s_mul_hi_u32 s9, s4, s12
+; GFX9-NEXT: s_mul_i32 s12, s9, s11
+; GFX9-NEXT: s_sub_i32 s4, s4, s12
+; GFX9-NEXT: s_cmp_ge_u32 s4, s11
; GFX9-NEXT: s_cselect_b32 s12, 1, 0
-; GFX9-NEXT: s_add_i32 s14, s11, 1
+; GFX9-NEXT: s_add_i32 s16, s9, 1
; GFX9-NEXT: s_cmp_lg_u32 s12, 0
-; GFX9-NEXT: s_cselect_b32 s11, s14, s11
-; GFX9-NEXT: s_sub_i32 s14, s1, s5
+; GFX9-NEXT: s_cselect_b32 s9, s16, s9
+; GFX9-NEXT: s_sub_i32 s16, s4, s11
; GFX9-NEXT: s_cmp_lg_u32 s12, 0
-; GFX9-NEXT: s_cselect_b32 s1, s14, s1
-; GFX9-NEXT: s_cmp_ge_u32 s1, s5
+; GFX9-NEXT: s_cselect_b32 s4, s16, s4
+; GFX9-NEXT: s_cmp_ge_u32 s4, s11
; GFX9-NEXT: s_cselect_b32 s12, 1, 0
-; GFX9-NEXT: s_add_i32 s14, s11, 1
+; GFX9-NEXT: s_add_i32 s16, s9, 1
; GFX9-NEXT: s_cmp_lg_u32 s12, 0
-; GFX9-NEXT: s_cselect_b32 s11, s14, s11
-; GFX9-NEXT: s_sub_i32 s5, s1, s5
+; GFX9-NEXT: s_cselect_b32 s9, s16, s9
+; GFX9-NEXT: s_sub_i32 s11, s4, s11
; GFX9-NEXT: s_cmp_lg_u32 s12, 0
-; GFX9-NEXT: s_cselect_b32 s1, s5, s1
-; GFX9-NEXT: s_ashr_i32 s12, s6, 31
-; GFX9-NEXT: s_add_i32 s5, s6, s12
-; GFX9-NEXT: s_xor_b32 s6, s5, s12
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX9-NEXT: s_xor_b32 s5, s10, s13
-; GFX9-NEXT: s_xor_b32 s11, s11, s5
-; GFX9-NEXT: s_xor_b32 s14, s1, s10
+; GFX9-NEXT: s_cselect_b32 s4, s11, s4
+; GFX9-NEXT: s_ashr_i32 s11, s13, 31
+; GFX9-NEXT: s_add_i32 s12, s13, s11
+; GFX9-NEXT: s_xor_b32 s12, s12, s11
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s12
+; GFX9-NEXT: s_xor_b32 s10, s8, s10
+; GFX9-NEXT: s_xor_b32 s9, s9, s10
+; GFX9-NEXT: s_xor_b32 s16, s4, s8
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT: s_sub_i32 s1, s11, s5
-; GFX9-NEXT: s_sub_i32 s5, s14, s10
-; GFX9-NEXT: s_sub_i32 s11, 0, s6
+; GFX9-NEXT: s_sub_i32 s4, s9, s10
+; GFX9-NEXT: s_sub_i32 s9, 0, s12
+; GFX9-NEXT: s_ashr_i32 s13, s5, 31
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: s_ashr_i32 s13, s2, 31
-; GFX9-NEXT: s_add_i32 s2, s2, s13
-; GFX9-NEXT: s_xor_b32 s2, s2, s13
+; GFX9-NEXT: s_add_i32 s5, s5, s13
+; GFX9-NEXT: s_xor_b32 s5, s5, s13
+; GFX9-NEXT: s_sub_i32 s8, s16, s8
; GFX9-NEXT: v_readfirstlane_b32 s10, v0
-; GFX9-NEXT: s_mul_i32 s11, s11, s10
-; GFX9-NEXT: s_mul_hi_u32 s11, s10, s11
-; GFX9-NEXT: s_add_i32 s10, s10, s11
-; GFX9-NEXT: s_mul_hi_u32 s10, s2, s10
-; GFX9-NEXT: s_mul_i32 s11, s10, s6
-; GFX9-NEXT: s_sub_i32 s2, s2, s11
-; GFX9-NEXT: s_cmp_ge_u32 s2, s6
-; GFX9-NEXT: s_cselect_b32 s11, 1, 0
-; GFX9-NEXT: s_add_i32 s14, s10, 1
-; GFX9-NEXT: s_cmp_lg_u32 s11, 0
-; GFX9-NEXT: s_cselect_b32 s10, s14, s10
-; GFX9-NEXT: s_sub_i32 s14, s2, s6
-; GFX9-NEXT: s_cmp_lg_u32 s11, 0
-; GFX9-NEXT: s_cselect_b32 s2, s14, s2
-; GFX9-NEXT: s_cmp_ge_u32 s2, s6
-; GFX9-NEXT: s_cselect_b32 s11, 1, 0
-; GFX9-NEXT: s_add_i32 s14, s10, 1
-; GFX9-NEXT: s_cmp_lg_u32 s11, 0
-; GFX9-NEXT: s_cselect_b32 s14, s14, s10
-; GFX9-NEXT: s_sub_i32 s6, s2, s6
-; GFX9-NEXT: s_cmp_lg_u32 s11, 0
-; GFX9-NEXT: s_cselect_b32 s6, s6, s2
-; GFX9-NEXT: s_ashr_i32 s15, s7, 31
-; GFX9-NEXT: s_add_i32 s2, s7, s15
-; GFX9-NEXT: s_xor_b32 s7, s2, s15
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_xor_b32 s2, s13, s12
-; GFX9-NEXT: s_xor_b32 s6, s6, s13
-; GFX9-NEXT: s_xor_b32 s12, s14, s2
+; GFX9-NEXT: s_mul_i32 s9, s9, s10
+; GFX9-NEXT: s_mul_hi_u32 s9, s10, s9
+; GFX9-NEXT: s_add_i32 s10, s10, s9
+; GFX9-NEXT: s_mul_hi_u32 s9, s5, s10
+; GFX9-NEXT: s_mul_i32 s10, s9, s12
+; GFX9-NEXT: s_sub_i32 s5, s5, s10
+; GFX9-NEXT: s_cmp_ge_u32 s5, s12
+; GFX9-NEXT: s_cselect_b32 s10, 1, 0
+; GFX9-NEXT: s_add_i32 s16, s9, 1
+; GFX9-NEXT: s_cmp_lg_u32 s10, 0
+; GFX9-NEXT: s_cselect_b32 s9, s16, s9
+; GFX9-NEXT: s_sub_i32 s16, s5, s12
+; GFX9-NEXT: s_cmp_lg_u32 s10, 0
+; GFX9-NEXT: s_cselect_b32 s5, s16, s5
+; GFX9-NEXT: s_cmp_ge_u32 s5, s12
+; GFX9-NEXT: s_cselect_b32 s10, 1, 0
+; GFX9-NEXT: s_add_i32 s16, s9, 1
+; GFX9-NEXT: s_cmp_lg_u32 s10, 0
+; GFX9-NEXT: s_cselect_b32 s9, s16, s9
+; GFX9-NEXT: s_sub_i32 s12, s5, s12
+; GFX9-NEXT: s_cmp_lg_u32 s10, 0
+; GFX9-NEXT: s_cselect_b32 s10, s12, s5
+; GFX9-NEXT: s_ashr_i32 s12, s14, 31
+; GFX9-NEXT: s_add_i32 s5, s14, s12
+; GFX9-NEXT: s_xor_b32 s14, s5, s12
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s14
+; GFX9-NEXT: s_xor_b32 s5, s13, s11
+; GFX9-NEXT: s_xor_b32 s9, s9, s5
+; GFX9-NEXT: s_sub_i32 s5, s9, s5
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT: s_xor_b32 s9, s10, s13
+; GFX9-NEXT: s_sub_i32 s9, s9, s13
+; GFX9-NEXT: s_sub_i32 s13, 0, s14
+; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX9-NEXT: s_ashr_i32 s10, s6, 31
+; GFX9-NEXT: s_add_i32 s6, s6, s10
+; GFX9-NEXT: s_xor_b32 s6, s6, s10
+; GFX9-NEXT: v_readfirstlane_b32 s11, v0
+; GFX9-NEXT: s_mul_i32 s13, s13, s11
+; GFX9-NEXT: s_mul_hi_u32 s13, s11, s13
+; GFX9-NEXT: s_add_i32 s11, s11, s13
+; GFX9-NEXT: s_mul_hi_u32 s11, s6, s11
+; GFX9-NEXT: s_mul_i32 s13, s11, s14
; GFX9-NEXT: s_sub_i32 s6, s6, s13
-; GFX9-NEXT: s_sub_i32 s14, 0, s7
-; GFX9-NEXT: s_sub_i32 s2, s12, s2
+; GFX9-NEXT: s_cmp_ge_u32 s6, s14
+; GFX9-NEXT: s_cselect_b32 s13, 1, 0
+; GFX9-NEXT: s_add_i32 s16, s11, 1
+; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: s_cselect_b32 s11, s16, s11
+; GFX9-NEXT: s_sub_i32 s16, s6, s14
+; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: s_cselect_b32 s6, s16, s6
+; GFX9-NEXT: s_cmp_ge_u32 s6, s14
+; GFX9-NEXT: s_cselect_b32 s13, 1, 0
+; GFX9-NEXT: s_add_i32 s16, s11, 1
+; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: s_cselect_b32 s11, s16, s11
+; GFX9-NEXT: s_sub_i32 s14, s6, s14
+; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: s_cselect_b32 s13, s14, s6
+; GFX9-NEXT: s_ashr_i32 s14, s15, 31
+; GFX9-NEXT: s_add_i32 s6, s15, s14
+; GFX9-NEXT: s_xor_b32 s15, s6, s14
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s15
+; GFX9-NEXT: s_xor_b32 s6, s10, s12
+; GFX9-NEXT: s_xor_b32 s11, s11, s6
+; GFX9-NEXT: s_sub_i32 s6, s11, s6
+; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX9-NEXT: s_xor_b32 s11, s13, s10
+; GFX9-NEXT: s_sub_i32 s13, 0, s15
+; GFX9-NEXT: s_sub_i32 s10, s11, s10
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: s_ashr_i32 s12, s3, 31
-; GFX9-NEXT: s_add_i32 s3, s3, s12
-; GFX9-NEXT: s_xor_b32 s3, s3, s12
-; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_mul_i32 s14, s14, s13
-; GFX9-NEXT: s_mul_hi_u32 s14, s13, s14
-; GFX9-NEXT: s_add_i32 s13, s13, s14
-; GFX9-NEXT: s_mul_hi_u32 s13, s3, s13
-; GFX9-NEXT: s_mul_i32 s14, s13, s7
-; GFX9-NEXT: s_sub_i32 s3, s3, s14
-; GFX9-NEXT: s_cmp_ge_u32 s3, s7
-; GFX9-NEXT: s_cselect_b32 s14, 1, 0
-; GFX9-NEXT: s_add_i32 s16, s13, 1
-; GFX9-NEXT: s_cmp_lg_u32 s14, 0
-; GFX9-NEXT: s_cselect_b32 s13, s16, s13
-; GFX9-NEXT: s_sub_i32 s16, s3, s7
-; GFX9-NEXT: s_cmp_lg_u32 s14, 0
-; GFX9-NEXT: s_cselect_b32 s3, s16, s3
-; GFX9-NEXT: s_cmp_ge_u32 s3, s7
-; GFX9-NEXT: s_cselect_b32 s14, 1, 0
-; GFX9-NEXT: s_add_i32 s16, s13, 1
-; GFX9-NEXT: s_cmp_lg_u32 s14, 0
-; GFX9-NEXT: s_cselect_b32 s13, s16, s13
-; GFX9-NEXT: s_sub_i32 s7, s3, s7
-; GFX9-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX9-NEXT: s_cmp_lg_u32 s14, 0
-; GFX9-NEXT: s_cselect_b32 s7, s7, s3
-; GFX9-NEXT: s_xor_b32 s3, s12, s15
-; GFX9-NEXT: s_xor_b32 s13, s13, s3
-; GFX9-NEXT: s_sub_i32 s3, s13, s3
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: s_xor_b32 s7, s7, s12
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: s_sub_i32 s7, s7, s12
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9]
-; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: s_ashr_i32 s11, s7, 31
+; GFX9-NEXT: s_add_i32 s7, s7, s11
+; GFX9-NEXT: s_xor_b32 s7, s7, s11
+; GFX9-NEXT: v_readfirstlane_b32 s12, v0
+; GFX9-NEXT: s_mul_i32 s13, s13, s12
+; GFX9-NEXT: s_mul_hi_u32 s13, s12, s13
+; GFX9-NEXT: s_add_i32 s12, s12, s13
+; GFX9-NEXT: s_mul_hi_u32 s12, s7, s12
+; GFX9-NEXT: s_mul_i32 s13, s12, s15
+; GFX9-NEXT: s_sub_i32 s7, s7, s13
+; GFX9-NEXT: s_cmp_ge_u32 s7, s15
+; GFX9-NEXT: s_cselect_b32 s13, 1, 0
+; GFX9-NEXT: s_add_i32 s16, s12, 1
+; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: s_cselect_b32 s12, s16, s12
+; GFX9-NEXT: s_sub_i32 s16, s7, s15
+; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: s_cselect_b32 s7, s16, s7
+; GFX9-NEXT: s_cmp_ge_u32 s7, s15
+; GFX9-NEXT: s_cselect_b32 s13, 1, 0
+; GFX9-NEXT: s_add_i32 s16, s12, 1
+; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: s_cselect_b32 s12, s16, s12
+; GFX9-NEXT: s_sub_i32 s15, s7, s15
+; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: s_cselect_b32 s13, s15, s7
+; GFX9-NEXT: s_xor_b32 s7, s11, s14
+; GFX9-NEXT: s_xor_b32 s12, s12, s7
+; GFX9-NEXT: s_sub_i32 s7, s12, s7
; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: s_xor_b32 s12, s13, s11
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[10:11]
+; GFX9-NEXT: s_sub_i32 s11, s12, s11
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-NEXT: v_mov_b32_e32 v1, s9
+; GFX9-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-NEXT: v_mov_b32_e32 v3, s11
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: sdivrem_v4i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
+; GFX10-NEXT: s_load_dwordx4 s[12:15], s[8:9], 0x20
; GFX10-NEXT: v_mov_b32_e32 v8, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_ashr_i32 s10, s4, 31
-; GFX10-NEXT: s_ashr_i32 s13, s0, 31
-; GFX10-NEXT: s_add_i32 s4, s4, s10
-; GFX10-NEXT: s_add_i32 s0, s0, s13
-; GFX10-NEXT: s_xor_b32 s4, s4, s10
-; GFX10-NEXT: s_xor_b32 s0, s0, s13
-; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s4
-; GFX10-NEXT: s_sub_i32 s12, 0, s4
+; GFX10-NEXT: s_ashr_i32 s10, s12, 31
+; GFX10-NEXT: s_add_i32 s0, s12, s10
+; GFX10-NEXT: s_xor_b32 s11, s0, s10
+; GFX10-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
+; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s11
+; GFX10-NEXT: s_sub_i32 s9, 0, s11
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_ashr_i32 s12, s4, 31
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s11, v0
-; GFX10-NEXT: s_mul_i32 s12, s12, s11
-; GFX10-NEXT: s_mul_hi_u32 s12, s11, s12
-; GFX10-NEXT: s_add_i32 s11, s11, s12
-; GFX10-NEXT: s_mul_hi_u32 s11, s0, s11
-; GFX10-NEXT: s_mul_i32 s12, s11, s4
-; GFX10-NEXT: s_sub_i32 s0, s0, s12
-; GFX10-NEXT: s_cmp_ge_u32 s0, s4
-; GFX10-NEXT: s_cselect_b32 s12, 1, 0
-; GFX10-NEXT: s_add_i32 s14, s11, 1
-; GFX10-NEXT: s_cmp_lg_u32 s12, 0
-; GFX10-NEXT: s_cselect_b32 s11, s14, s11
-; GFX10-NEXT: s_sub_i32 s14, s0, s4
-; GFX10-NEXT: s_cmp_lg_u32 s12, 0
-; GFX10-NEXT: s_cselect_b32 s0, s14, s0
-; GFX10-NEXT: s_cmp_ge_u32 s0, s4
-; GFX10-NEXT: s_cselect_b32 s12, 1, 0
-; GFX10-NEXT: s_add_i32 s14, s11, 1
-; GFX10-NEXT: s_cmp_lg_u32 s12, 0
-; GFX10-NEXT: s_cselect_b32 s11, s14, s11
-; GFX10-NEXT: s_sub_i32 s4, s0, s4
-; GFX10-NEXT: s_cmp_lg_u32 s12, 0
-; GFX10-NEXT: s_cselect_b32 s0, s4, s0
-; GFX10-NEXT: s_ashr_i32 s12, s5, 31
-; GFX10-NEXT: s_ashr_i32 s15, s1, 31
-; GFX10-NEXT: s_add_i32 s4, s5, s12
-; GFX10-NEXT: s_add_i32 s1, s1, s15
-; GFX10-NEXT: s_xor_b32 s5, s4, s12
-; GFX10-NEXT: s_xor_b32 s1, s1, s15
-; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX10-NEXT: s_sub_i32 s14, 0, s5
-; GFX10-NEXT: s_xor_b32 s10, s13, s10
-; GFX10-NEXT: s_xor_b32 s11, s11, s10
+; GFX10-NEXT: s_add_i32 s4, s4, s12
+; GFX10-NEXT: s_xor_b32 s4, s4, s12
+; GFX10-NEXT: v_readfirstlane_b32 s8, v0
+; GFX10-NEXT: s_mul_i32 s9, s9, s8
+; GFX10-NEXT: s_mul_hi_u32 s9, s8, s9
+; GFX10-NEXT: s_add_i32 s8, s8, s9
+; GFX10-NEXT: s_mul_hi_u32 s8, s4, s8
+; GFX10-NEXT: s_mul_i32 s9, s8, s11
+; GFX10-NEXT: s_sub_i32 s4, s4, s9
+; GFX10-NEXT: s_cmp_ge_u32 s4, s11
+; GFX10-NEXT: s_cselect_b32 s9, 1, 0
+; GFX10-NEXT: s_add_i32 s16, s8, 1
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_cselect_b32 s8, s16, s8
+; GFX10-NEXT: s_sub_i32 s16, s4, s11
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_cselect_b32 s4, s16, s4
+; GFX10-NEXT: s_cmp_ge_u32 s4, s11
+; GFX10-NEXT: s_cselect_b32 s9, 1, 0
+; GFX10-NEXT: s_add_i32 s16, s8, 1
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_cselect_b32 s8, s16, s8
+; GFX10-NEXT: s_sub_i32 s11, s4, s11
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: s_cselect_b32 s4, s11, s4
+; GFX10-NEXT: s_ashr_i32 s9, s13, 31
+; GFX10-NEXT: s_ashr_i32 s17, s5, 31
+; GFX10-NEXT: s_add_i32 s11, s13, s9
+; GFX10-NEXT: s_add_i32 s5, s5, s17
+; GFX10-NEXT: s_xor_b32 s11, s11, s9
+; GFX10-NEXT: s_xor_b32 s5, s5, s17
+; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s11
+; GFX10-NEXT: s_sub_i32 s16, 0, s11
+; GFX10-NEXT: s_xor_b32 s10, s12, s10
+; GFX10-NEXT: s_xor_b32 s8, s8, s10
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s4, v0
-; GFX10-NEXT: s_mul_i32 s14, s14, s4
-; GFX10-NEXT: s_mul_hi_u32 s14, s4, s14
-; GFX10-NEXT: s_add_i32 s4, s4, s14
-; GFX10-NEXT: s_mul_hi_u32 s14, s1, s4
-; GFX10-NEXT: s_xor_b32 s4, s0, s13
-; GFX10-NEXT: s_mul_i32 s16, s14, s5
-; GFX10-NEXT: s_sub_i32 s0, s11, s10
-; GFX10-NEXT: s_sub_i32 s1, s1, s16
-; GFX10-NEXT: s_sub_i32 s4, s4, s13
-; GFX10-NEXT: s_cmp_ge_u32 s1, s5
+; GFX10-NEXT: v_readfirstlane_b32 s13, v0
+; GFX10-NEXT: s_mul_i32 s16, s16, s13
+; GFX10-NEXT: s_mul_hi_u32 s16, s13, s16
+; GFX10-NEXT: s_add_i32 s13, s13, s16
+; GFX10-NEXT: s_xor_b32 s16, s4, s12
+; GFX10-NEXT: s_mul_hi_u32 s13, s5, s13
+; GFX10-NEXT: s_sub_i32 s4, s8, s10
+; GFX10-NEXT: s_mul_i32 s18, s13, s11
+; GFX10-NEXT: s_sub_i32 s8, s16, s12
+; GFX10-NEXT: s_sub_i32 s5, s5, s18
+; GFX10-NEXT: s_cmp_ge_u32 s5, s11
; GFX10-NEXT: s_cselect_b32 s10, 1, 0
-; GFX10-NEXT: s_add_i32 s11, s14, 1
+; GFX10-NEXT: s_add_i32 s12, s13, 1
; GFX10-NEXT: s_cmp_lg_u32 s10, 0
-; GFX10-NEXT: s_cselect_b32 s11, s11, s14
-; GFX10-NEXT: s_sub_i32 s13, s1, s5
+; GFX10-NEXT: s_cselect_b32 s12, s12, s13
+; GFX10-NEXT: s_sub_i32 s13, s5, s11
; GFX10-NEXT: s_cmp_lg_u32 s10, 0
-; GFX10-NEXT: s_cselect_b32 s1, s13, s1
-; GFX10-NEXT: s_cmp_ge_u32 s1, s5
+; GFX10-NEXT: s_cselect_b32 s5, s13, s5
+; GFX10-NEXT: s_cmp_ge_u32 s5, s11
; GFX10-NEXT: s_cselect_b32 s10, 1, 0
-; GFX10-NEXT: s_add_i32 s13, s11, 1
+; GFX10-NEXT: s_add_i32 s13, s12, 1
; GFX10-NEXT: s_cmp_lg_u32 s10, 0
-; GFX10-NEXT: s_cselect_b32 s11, s13, s11
-; GFX10-NEXT: s_sub_i32 s5, s1, s5
+; GFX10-NEXT: s_cselect_b32 s12, s13, s12
+; GFX10-NEXT: s_sub_i32 s11, s5, s11
; GFX10-NEXT: s_cmp_lg_u32 s10, 0
-; GFX10-NEXT: s_cselect_b32 s1, s5, s1
-; GFX10-NEXT: s_ashr_i32 s10, s6, 31
-; GFX10-NEXT: s_ashr_i32 s14, s2, 31
-; GFX10-NEXT: s_add_i32 s5, s6, s10
-; GFX10-NEXT: s_add_i32 s2, s2, s14
-; GFX10-NEXT: s_xor_b32 s6, s5, s10
-; GFX10-NEXT: s_xor_b32 s2, s2, s14
-; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX10-NEXT: s_sub_i32 s13, 0, s6
-; GFX10-NEXT: s_xor_b32 s12, s15, s12
-; GFX10-NEXT: s_xor_b32 s11, s11, s12
+; GFX10-NEXT: s_cselect_b32 s5, s11, s5
+; GFX10-NEXT: s_ashr_i32 s10, s14, 31
+; GFX10-NEXT: s_ashr_i32 s16, s6, 31
+; GFX10-NEXT: s_add_i32 s11, s14, s10
+; GFX10-NEXT: s_add_i32 s6, s6, s16
+; GFX10-NEXT: s_xor_b32 s11, s11, s10
+; GFX10-NEXT: s_xor_b32 s6, s6, s16
+; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s11
+; GFX10-NEXT: s_sub_i32 s14, 0, s11
+; GFX10-NEXT: s_xor_b32 s9, s17, s9
+; GFX10-NEXT: s_xor_b32 s12, s12, s9
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s5, v0
-; GFX10-NEXT: s_mul_i32 s13, s13, s5
-; GFX10-NEXT: s_mul_hi_u32 s13, s5, s13
-; GFX10-NEXT: s_add_i32 s5, s5, s13
-; GFX10-NEXT: s_mul_hi_u32 s13, s2, s5
-; GFX10-NEXT: s_xor_b32 s5, s1, s15
-; GFX10-NEXT: s_mul_i32 s16, s13, s6
-; GFX10-NEXT: s_sub_i32 s1, s11, s12
-; GFX10-NEXT: s_sub_i32 s2, s2, s16
-; GFX10-NEXT: s_sub_i32 s5, s5, s15
-; GFX10-NEXT: s_cmp_ge_u32 s2, s6
-; GFX10-NEXT: s_cselect_b32 s11, 1, 0
-; GFX10-NEXT: s_add_i32 s12, s13, 1
-; GFX10-NEXT: s_cmp_lg_u32 s11, 0
-; GFX10-NEXT: s_cselect_b32 s12, s12, s13
-; GFX10-NEXT: s_sub_i32 s13, s2, s6
-; GFX10-NEXT: s_cmp_lg_u32 s11, 0
-; GFX10-NEXT: s_cselect_b32 s2, s13, s2
-; GFX10-NEXT: s_cmp_ge_u32 s2, s6
-; GFX10-NEXT: s_cselect_b32 s11, 1, 0
-; GFX10-NEXT: s_add_i32 s13, s12, 1
-; GFX10-NEXT: s_cmp_lg_u32 s11, 0
-; GFX10-NEXT: s_cselect_b32 s12, s13, s12
-; GFX10-NEXT: s_sub_i32 s6, s2, s6
-; GFX10-NEXT: s_cmp_lg_u32 s11, 0
-; GFX10-NEXT: s_cselect_b32 s2, s6, s2
-; GFX10-NEXT: s_ashr_i32 s13, s7, 31
-; GFX10-NEXT: s_ashr_i32 s15, s3, 31
-; GFX10-NEXT: s_add_i32 s6, s7, s13
-; GFX10-NEXT: s_add_i32 s3, s3, s15
-; GFX10-NEXT: s_xor_b32 s7, s6, s13
-; GFX10-NEXT: s_xor_b32 s10, s14, s10
-; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX10-NEXT: s_sub_i32 s11, 0, s7
-; GFX10-NEXT: s_xor_b32 s3, s3, s15
+; GFX10-NEXT: v_readfirstlane_b32 s13, v0
+; GFX10-NEXT: s_mul_i32 s14, s14, s13
+; GFX10-NEXT: s_mul_hi_u32 s14, s13, s14
+; GFX10-NEXT: s_add_i32 s13, s13, s14
+; GFX10-NEXT: s_xor_b32 s14, s5, s17
+; GFX10-NEXT: s_mul_hi_u32 s13, s6, s13
+; GFX10-NEXT: s_sub_i32 s5, s12, s9
+; GFX10-NEXT: s_mul_i32 s18, s13, s11
+; GFX10-NEXT: s_sub_i32 s9, s14, s17
+; GFX10-NEXT: s_sub_i32 s6, s6, s18
+; GFX10-NEXT: s_cmp_ge_u32 s6, s11
+; GFX10-NEXT: s_cselect_b32 s12, 1, 0
+; GFX10-NEXT: s_add_i32 s14, s13, 1
+; GFX10-NEXT: s_cmp_lg_u32 s12, 0
+; GFX10-NEXT: s_cselect_b32 s13, s14, s13
+; GFX10-NEXT: s_sub_i32 s14, s6, s11
+; GFX10-NEXT: s_cmp_lg_u32 s12, 0
+; GFX10-NEXT: s_cselect_b32 s6, s14, s6
+; GFX10-NEXT: s_cmp_ge_u32 s6, s11
+; GFX10-NEXT: s_cselect_b32 s12, 1, 0
+; GFX10-NEXT: s_add_i32 s14, s13, 1
+; GFX10-NEXT: s_cmp_lg_u32 s12, 0
+; GFX10-NEXT: s_cselect_b32 s13, s14, s13
+; GFX10-NEXT: s_sub_i32 s11, s6, s11
+; GFX10-NEXT: s_cmp_lg_u32 s12, 0
+; GFX10-NEXT: s_cselect_b32 s6, s11, s6
+; GFX10-NEXT: s_ashr_i32 s11, s15, 31
+; GFX10-NEXT: s_ashr_i32 s17, s7, 31
+; GFX10-NEXT: s_add_i32 s12, s15, s11
+; GFX10-NEXT: s_add_i32 s7, s7, s17
+; GFX10-NEXT: s_xor_b32 s12, s12, s11
+; GFX10-NEXT: s_xor_b32 s7, s7, s17
+; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s12
+; GFX10-NEXT: s_sub_i32 s15, 0, s12
+; GFX10-NEXT: s_xor_b32 s10, s16, s10
+; GFX10-NEXT: s_xor_b32 s13, s13, s10
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s6, v0
-; GFX10-NEXT: s_mul_i32 s11, s11, s6
-; GFX10-NEXT: s_mul_hi_u32 s11, s6, s11
-; GFX10-NEXT: s_add_i32 s6, s6, s11
-; GFX10-NEXT: s_xor_b32 s11, s12, s10
-; GFX10-NEXT: s_mul_hi_u32 s12, s3, s6
-; GFX10-NEXT: s_xor_b32 s6, s2, s14
-; GFX10-NEXT: s_mul_i32 s16, s12, s7
-; GFX10-NEXT: s_sub_i32 s2, s11, s10
-; GFX10-NEXT: s_sub_i32 s3, s3, s16
-; GFX10-NEXT: s_sub_i32 s6, s6, s14
-; GFX10-NEXT: s_cmp_ge_u32 s3, s7
-; GFX10-NEXT: s_cselect_b32 s10, 1, 0
-; GFX10-NEXT: s_add_i32 s11, s12, 1
-; GFX10-NEXT: s_cmp_lg_u32 s10, 0
-; GFX10-NEXT: s_cselect_b32 s12, s11, s12
-; GFX10-NEXT: s_sub_i32 s11, s3, s7
-; GFX10-NEXT: s_cmp_lg_u32 s10, 0
-; GFX10-NEXT: s_cselect_b32 s3, s11, s3
-; GFX10-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX10-NEXT: s_cmp_ge_u32 s3, s7
-; GFX10-NEXT: s_cselect_b32 s14, 1, 0
-; GFX10-NEXT: s_add_i32 s16, s12, 1
-; GFX10-NEXT: s_cmp_lg_u32 s14, 0
-; GFX10-NEXT: s_cselect_b32 s12, s16, s12
-; GFX10-NEXT: s_sub_i32 s7, s3, s7
-; GFX10-NEXT: s_cmp_lg_u32 s14, 0
-; GFX10-NEXT: s_cselect_b32 s3, s7, s3
-; GFX10-NEXT: s_xor_b32 s7, s15, s13
-; GFX10-NEXT: s_xor_b32 s13, s3, s15
-; GFX10-NEXT: s_xor_b32 s12, s12, s7
-; GFX10-NEXT: s_sub_i32 s3, s12, s7
-; GFX10-NEXT: s_sub_i32 s7, s13, s15
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s5
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s7
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
-; GFX10-NEXT: global_store_dwordx4 v8, v[4:7], s[10:11]
+; GFX10-NEXT: v_readfirstlane_b32 s14, v0
+; GFX10-NEXT: s_mul_i32 s15, s15, s14
+; GFX10-NEXT: s_mul_hi_u32 s15, s14, s15
+; GFX10-NEXT: s_add_i32 s14, s14, s15
+; GFX10-NEXT: s_xor_b32 s15, s6, s16
+; GFX10-NEXT: s_mul_hi_u32 s14, s7, s14
+; GFX10-NEXT: s_sub_i32 s6, s13, s10
+; GFX10-NEXT: s_mul_i32 s18, s14, s12
+; GFX10-NEXT: s_sub_i32 s10, s15, s16
+; GFX10-NEXT: s_sub_i32 s7, s7, s18
+; GFX10-NEXT: s_cmp_ge_u32 s7, s12
+; GFX10-NEXT: s_cselect_b32 s13, 1, 0
+; GFX10-NEXT: s_add_i32 s15, s14, 1
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: s_cselect_b32 s14, s15, s14
+; GFX10-NEXT: s_sub_i32 s15, s7, s12
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: s_cselect_b32 s7, s15, s7
+; GFX10-NEXT: s_cmp_ge_u32 s7, s12
+; GFX10-NEXT: s_cselect_b32 s13, 1, 0
+; GFX10-NEXT: s_add_i32 s15, s14, 1
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: s_cselect_b32 s14, s15, s14
+; GFX10-NEXT: s_sub_i32 s12, s7, s12
+; GFX10-NEXT: s_cmp_lg_u32 s13, 0
+; GFX10-NEXT: s_cselect_b32 s7, s12, s7
+; GFX10-NEXT: s_xor_b32 s11, s17, s11
+; GFX10-NEXT: s_xor_b32 s13, s7, s17
+; GFX10-NEXT: s_xor_b32 s12, s14, s11
+; GFX10-NEXT: s_sub_i32 s7, s12, s11
+; GFX10-NEXT: s_sub_i32 s11, s13, s17
+; GFX10-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-NEXT: v_mov_b32_e32 v2, s6
+; GFX10-NEXT: v_mov_b32_e32 v3, s7
+; GFX10-NEXT: v_mov_b32_e32 v4, s8
+; GFX10-NEXT: v_mov_b32_e32 v5, s9
+; GFX10-NEXT: v_mov_b32_e32 v6, s10
+; GFX10-NEXT: v_mov_b32_e32 v7, s11
+; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX10-NEXT: global_store_dwordx4 v8, v[4:7], s[2:3]
; GFX10-NEXT: s_endpgm
%div = sdiv <4 x i32> %x, %y
store <4 x i32> %div, ptr addrspace(1) %out0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
index bad1a7c32ea26..84c1fb7ab9947 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/udivrem.ll
@@ -867,16 +867,18 @@ define amdgpu_kernel void @udivrem_v2i32(ptr addrspace(1) %out0, ptr addrspace(1
define amdgpu_kernel void @udivrem_v4i32(ptr addrspace(1) %out0, ptr addrspace(1) %out1, <4 x i32> %x, <4 x i32> %y) {
; GFX8-LABEL: udivrem_v4i32:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
; GFX8-NEXT: s_add_i32 s12, s12, s17
-; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX8-NEXT: s_load_dwordx4 s[12:15], s[8:9], 0x20
+; GFX8-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s4
-; GFX8-NEXT: s_sub_i32 s10, 0, s4
-; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s5
-; GFX8-NEXT: v_cvt_f32_u32_e32 v2, s6
+; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s12
+; GFX8-NEXT: s_sub_i32 s8, 0, s12
+; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s13
+; GFX8-NEXT: v_cvt_f32_u32_e32 v2, s14
; GFX8-NEXT: v_rcp_iflag_f32_e32 v0, v0
+; GFX8-NEXT: v_mov_b32_e32 v5, s1
; GFX8-NEXT: v_rcp_iflag_f32_e32 v1, v1
; GFX8-NEXT: v_rcp_iflag_f32_e32 v2, v2
; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
@@ -884,396 +886,393 @@ define amdgpu_kernel void @udivrem_v4i32(ptr addrspace(1) %out0, ptr addrspace(1
; GFX8-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX8-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
-; GFX8-NEXT: v_readfirstlane_b32 s11, v0
-; GFX8-NEXT: s_mul_i32 s10, s10, s11
-; GFX8-NEXT: v_mul_hi_u32 v0, v0, s10
+; GFX8-NEXT: v_readfirstlane_b32 s9, v0
+; GFX8-NEXT: s_mul_i32 s8, s8, s9
+; GFX8-NEXT: v_mul_hi_u32 v0, v0, s8
; GFX8-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: v_readfirstlane_b32 s8, v0
+; GFX8-NEXT: s_add_i32 s8, s9, s8
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mul_hi_u32 v0, s4, v0
+; GFX8-NEXT: v_readfirstlane_b32 s9, v1
+; GFX8-NEXT: v_readfirstlane_b32 s8, v0
+; GFX8-NEXT: s_mul_i32 s10, s8, s12
+; GFX8-NEXT: s_sub_i32 s4, s4, s10
+; GFX8-NEXT: s_cmp_ge_u32 s4, s12
+; GFX8-NEXT: s_cselect_b32 s10, 1, 0
+; GFX8-NEXT: s_add_i32 s11, s8, 1
+; GFX8-NEXT: s_cmp_lg_u32 s10, 0
+; GFX8-NEXT: s_cselect_b32 s8, s11, s8
+; GFX8-NEXT: s_sub_i32 s11, s4, s12
+; GFX8-NEXT: s_cmp_lg_u32 s10, 0
+; GFX8-NEXT: s_cselect_b32 s4, s11, s4
+; GFX8-NEXT: s_cmp_ge_u32 s4, s12
+; GFX8-NEXT: s_cselect_b32 s10, 1, 0
+; GFX8-NEXT: s_add_i32 s11, s8, 1
+; GFX8-NEXT: s_cmp_lg_u32 s10, 0
+; GFX8-NEXT: s_cselect_b32 s8, s11, s8
+; GFX8-NEXT: s_sub_i32 s11, s4, s12
+; GFX8-NEXT: s_cmp_lg_u32 s10, 0
+; GFX8-NEXT: s_cselect_b32 s4, s11, s4
+; GFX8-NEXT: s_sub_i32 s10, 0, s13
+; GFX8-NEXT: s_mul_i32 s10, s10, s9
+; GFX8-NEXT: v_mul_hi_u32 v0, v1, s10
+; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s15
; GFX8-NEXT: v_readfirstlane_b32 s10, v0
-; GFX8-NEXT: s_add_i32 s10, s11, s10
-; GFX8-NEXT: v_mov_b32_e32 v0, s10
-; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0
-; GFX8-NEXT: v_readfirstlane_b32 s10, v1
-; GFX8-NEXT: v_readfirstlane_b32 s11, v0
-; GFX8-NEXT: s_mul_i32 s12, s11, s4
-; GFX8-NEXT: s_sub_i32 s0, s0, s12
-; GFX8-NEXT: s_cmp_ge_u32 s0, s4
-; GFX8-NEXT: s_cselect_b32 s12, 1, 0
-; GFX8-NEXT: s_add_i32 s13, s11, 1
-; GFX8-NEXT: s_cmp_lg_u32 s12, 0
-; GFX8-NEXT: s_cselect_b32 s11, s13, s11
-; GFX8-NEXT: s_sub_i32 s13, s0, s4
-; GFX8-NEXT: s_cmp_lg_u32 s12, 0
-; GFX8-NEXT: s_cselect_b32 s0, s13, s0
-; GFX8-NEXT: s_cmp_ge_u32 s0, s4
-; GFX8-NEXT: s_cselect_b32 s13, 1, 0
-; GFX8-NEXT: s_add_i32 s12, s11, 1
-; GFX8-NEXT: s_cmp_lg_u32 s13, 0
-; GFX8-NEXT: s_cselect_b32 s12, s12, s11
-; GFX8-NEXT: s_sub_i32 s4, s0, s4
-; GFX8-NEXT: s_cmp_lg_u32 s13, 0
-; GFX8-NEXT: s_cselect_b32 s0, s4, s0
-; GFX8-NEXT: s_sub_i32 s4, 0, s5
-; GFX8-NEXT: s_mul_i32 s4, s4, s10
-; GFX8-NEXT: v_mul_hi_u32 v0, v1, s4
-; GFX8-NEXT: v_cvt_f32_u32_e32 v1, s7
-; GFX8-NEXT: v_readfirstlane_b32 s4, v0
-; GFX8-NEXT: s_add_i32 s4, s10, s4
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mul_hi_u32 v0, s1, v0
-; GFX8-NEXT: v_readfirstlane_b32 s4, v2
+; GFX8-NEXT: s_add_i32 s9, s9, s10
+; GFX8-NEXT: v_mov_b32_e32 v0, s9
+; GFX8-NEXT: v_mul_hi_u32 v0, s5, v0
+; GFX8-NEXT: v_readfirstlane_b32 s10, v2
; GFX8-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GFX8-NEXT: v_readfirstlane_b32 s10, v0
-; GFX8-NEXT: s_mul_i32 s11, s10, s5
-; GFX8-NEXT: s_sub_i32 s1, s1, s11
-; GFX8-NEXT: s_cmp_ge_u32 s1, s5
+; GFX8-NEXT: v_readfirstlane_b32 s9, v0
+; GFX8-NEXT: s_mul_i32 s11, s9, s13
+; GFX8-NEXT: s_sub_i32 s5, s5, s11
+; GFX8-NEXT: s_cmp_ge_u32 s5, s13
; GFX8-NEXT: s_cselect_b32 s11, 1, 0
-; GFX8-NEXT: s_add_i32 s13, s10, 1
+; GFX8-NEXT: s_add_i32 s12, s9, 1
; GFX8-NEXT: s_cmp_lg_u32 s11, 0
-; GFX8-NEXT: s_cselect_b32 s10, s13, s10
-; GFX8-NEXT: s_sub_i32 s13, s1, s5
+; GFX8-NEXT: s_cselect_b32 s9, s12, s9
+; GFX8-NEXT: s_sub_i32 s12, s5, s13
; GFX8-NEXT: s_cmp_lg_u32 s11, 0
-; GFX8-NEXT: s_cselect_b32 s1, s13, s1
-; GFX8-NEXT: s_cmp_ge_u32 s1, s5
+; GFX8-NEXT: s_cselect_b32 s5, s12, s5
+; GFX8-NEXT: s_cmp_ge_u32 s5, s13
; GFX8-NEXT: s_cselect_b32 s11, 1, 0
-; GFX8-NEXT: s_add_i32 s13, s10, 1
+; GFX8-NEXT: s_add_i32 s12, s9, 1
; GFX8-NEXT: s_cmp_lg_u32 s11, 0
-; GFX8-NEXT: s_cselect_b32 s13, s13, s10
-; GFX8-NEXT: s_sub_i32 s5, s1, s5
+; GFX8-NEXT: s_cselect_b32 s9, s12, s9
+; GFX8-NEXT: s_sub_i32 s12, s5, s13
; GFX8-NEXT: s_cmp_lg_u32 s11, 0
-; GFX8-NEXT: s_cselect_b32 s1, s5, s1
-; GFX8-NEXT: s_sub_i32 s5, 0, s6
-; GFX8-NEXT: s_mul_i32 s5, s5, s4
-; GFX8-NEXT: v_mul_hi_u32 v0, v2, s5
+; GFX8-NEXT: s_cselect_b32 s5, s12, s5
+; GFX8-NEXT: s_sub_i32 s11, 0, s14
+; GFX8-NEXT: s_mul_i32 s11, s11, s10
+; GFX8-NEXT: v_mul_hi_u32 v0, v2, s11
; GFX8-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX8-NEXT: v_readfirstlane_b32 s5, v0
-; GFX8-NEXT: s_add_i32 s4, s4, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
-; GFX8-NEXT: v_readfirstlane_b32 s4, v1
-; GFX8-NEXT: v_readfirstlane_b32 s5, v0
-; GFX8-NEXT: s_mul_i32 s10, s5, s6
-; GFX8-NEXT: s_sub_i32 s2, s2, s10
-; GFX8-NEXT: s_cmp_ge_u32 s2, s6
-; GFX8-NEXT: s_cselect_b32 s10, 1, 0
-; GFX8-NEXT: s_add_i32 s11, s5, 1
-; GFX8-NEXT: s_cmp_lg_u32 s10, 0
-; GFX8-NEXT: s_cselect_b32 s5, s11, s5
-; GFX8-NEXT: s_sub_i32 s11, s2, s6
-; GFX8-NEXT: s_cmp_lg_u32 s10, 0
-; GFX8-NEXT: s_cselect_b32 s2, s11, s2
-; GFX8-NEXT: s_cmp_ge_u32 s2, s6
-; GFX8-NEXT: s_cselect_b32 s10, 1, 0
-; GFX8-NEXT: s_add_i32 s11, s5, 1
-; GFX8-NEXT: s_cmp_lg_u32 s10, 0
-; GFX8-NEXT: s_cselect_b32 s14, s11, s5
-; GFX8-NEXT: s_sub_i32 s5, s2, s6
-; GFX8-NEXT: s_cmp_lg_u32 s10, 0
-; GFX8-NEXT: s_cselect_b32 s2, s5, s2
-; GFX8-NEXT: s_sub_i32 s5, 0, s7
-; GFX8-NEXT: s_mul_i32 s5, s5, s4
-; GFX8-NEXT: v_mul_hi_u32 v0, v1, s5
-; GFX8-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX8-NEXT: v_readfirstlane_b32 s5, v0
-; GFX8-NEXT: s_add_i32 s4, s4, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mul_hi_u32 v0, s3, v0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, s8
-; GFX8-NEXT: v_mov_b32_e32 v5, s9
-; GFX8-NEXT: v_readfirstlane_b32 s4, v0
-; GFX8-NEXT: s_mul_i32 s5, s4, s7
-; GFX8-NEXT: s_sub_i32 s3, s3, s5
-; GFX8-NEXT: s_cmp_ge_u32 s3, s7
-; GFX8-NEXT: s_cselect_b32 s5, 1, 0
-; GFX8-NEXT: s_add_i32 s6, s4, 1
-; GFX8-NEXT: s_cmp_lg_u32 s5, 0
-; GFX8-NEXT: s_cselect_b32 s4, s6, s4
-; GFX8-NEXT: s_sub_i32 s6, s3, s7
-; GFX8-NEXT: s_cmp_lg_u32 s5, 0
-; GFX8-NEXT: s_cselect_b32 s3, s6, s3
-; GFX8-NEXT: s_cmp_ge_u32 s3, s7
-; GFX8-NEXT: s_cselect_b32 s5, 1, 0
-; GFX8-NEXT: s_add_i32 s6, s4, 1
-; GFX8-NEXT: s_cmp_lg_u32 s5, 0
-; GFX8-NEXT: s_cselect_b32 s15, s6, s4
-; GFX8-NEXT: s_sub_i32 s4, s3, s7
-; GFX8-NEXT: v_mov_b32_e32 v0, s12
-; GFX8-NEXT: v_mov_b32_e32 v1, s13
-; GFX8-NEXT: v_mov_b32_e32 v2, s14
-; GFX8-NEXT: v_mov_b32_e32 v3, s15
-; GFX8-NEXT: s_cmp_lg_u32 s5, 0
-; GFX8-NEXT: s_cselect_b32 s3, s4, s3
+; GFX8-NEXT: v_readfirstlane_b32 s11, v0
+; GFX8-NEXT: s_add_i32 s10, s10, s11
+; GFX8-NEXT: v_mov_b32_e32 v0, s10
+; GFX8-NEXT: v_mul_hi_u32 v0, s6, v0
+; GFX8-NEXT: v_readfirstlane_b32 s11, v1
+; GFX8-NEXT: v_readfirstlane_b32 s10, v0
+; GFX8-NEXT: s_mul_i32 s12, s10, s14
+; GFX8-NEXT: s_sub_i32 s6, s6, s12
+; GFX8-NEXT: s_cmp_ge_u32 s6, s14
+; GFX8-NEXT: s_cselect_b32 s12, 1, 0
+; GFX8-NEXT: s_add_i32 s13, s10, 1
+; GFX8-NEXT: s_cmp_lg_u32 s12, 0
+; GFX8-NEXT: s_cselect_b32 s10, s13, s10
+; GFX8-NEXT: s_sub_i32 s13, s6, s14
+; GFX8-NEXT: s_cmp_lg_u32 s12, 0
+; GFX8-NEXT: s_cselect_b32 s6, s13, s6
+; GFX8-NEXT: s_cmp_ge_u32 s6, s14
+; GFX8-NEXT: s_cselect_b32 s12, 1, 0
+; GFX8-NEXT: s_add_i32 s13, s10, 1
+; GFX8-NEXT: s_cmp_lg_u32 s12, 0
+; GFX8-NEXT: s_cselect_b32 s10, s13, s10
+; GFX8-NEXT: s_sub_i32 s13, s6, s14
+; GFX8-NEXT: s_cmp_lg_u32 s12, 0
+; GFX8-NEXT: s_cselect_b32 s6, s13, s6
+; GFX8-NEXT: s_sub_i32 s12, 0, s15
+; GFX8-NEXT: s_mul_i32 s12, s12, s11
+; GFX8-NEXT: v_mul_hi_u32 v0, v1, s12
+; GFX8-NEXT: v_readfirstlane_b32 s12, v0
+; GFX8-NEXT: s_add_i32 s11, s11, s12
+; GFX8-NEXT: v_mov_b32_e32 v0, s11
+; GFX8-NEXT: v_mul_hi_u32 v0, s7, v0
+; GFX8-NEXT: v_readfirstlane_b32 s11, v0
+; GFX8-NEXT: s_mul_i32 s12, s11, s15
+; GFX8-NEXT: s_sub_i32 s7, s7, s12
+; GFX8-NEXT: s_cmp_ge_u32 s7, s15
+; GFX8-NEXT: s_cselect_b32 s12, 1, 0
+; GFX8-NEXT: s_add_i32 s13, s11, 1
+; GFX8-NEXT: s_cmp_lg_u32 s12, 0
+; GFX8-NEXT: s_cselect_b32 s11, s13, s11
+; GFX8-NEXT: s_sub_i32 s13, s7, s15
+; GFX8-NEXT: s_cmp_lg_u32 s12, 0
+; GFX8-NEXT: s_cselect_b32 s7, s13, s7
+; GFX8-NEXT: s_cmp_ge_u32 s7, s15
+; GFX8-NEXT: s_cselect_b32 s12, 1, 0
+; GFX8-NEXT: s_add_i32 s13, s11, 1
+; GFX8-NEXT: s_cmp_lg_u32 s12, 0
+; GFX8-NEXT: s_cselect_b32 s11, s13, s11
+; GFX8-NEXT: s_sub_i32 s13, s7, s15
+; GFX8-NEXT: v_mov_b32_e32 v0, s8
+; GFX8-NEXT: v_mov_b32_e32 v1, s9
+; GFX8-NEXT: v_mov_b32_e32 v2, s10
+; GFX8-NEXT: v_mov_b32_e32 v3, s11
+; GFX8-NEXT: s_cmp_lg_u32 s12, 0
+; GFX8-NEXT: s_cselect_b32 s7, s13, s7
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GFX8-NEXT: v_mov_b32_e32 v4, s10
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
-; GFX8-NEXT: v_mov_b32_e32 v5, s11
+; GFX8-NEXT: v_mov_b32_e32 v5, s3
+; GFX8-NEXT: v_mov_b32_e32 v0, s4
+; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
+; GFX8-NEXT: v_mov_b32_e32 v3, s7
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
; GFX8-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: udivrem_v4i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
+; GFX9-NEXT: s_load_dwordx4 s[12:15], s[8:9], 0x20
+; GFX9-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4
-; GFX9-NEXT: s_sub_i32 s12, 0, s4
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s5
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s12
+; GFX9-NEXT: s_sub_i32 s8, 0, s12
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s13
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX9-NEXT: v_rcp_iflag_f32_e32 v1, v1
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v1
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX9-NEXT: v_readfirstlane_b32 s13, v0
-; GFX9-NEXT: s_mul_i32 s12, s12, s13
-; GFX9-NEXT: s_mul_hi_u32 s12, s13, s12
-; GFX9-NEXT: s_add_i32 s13, s13, s12
-; GFX9-NEXT: s_mul_hi_u32 s12, s0, s13
-; GFX9-NEXT: s_mul_i32 s13, s12, s4
-; GFX9-NEXT: s_sub_i32 s0, s0, s13
-; GFX9-NEXT: s_cmp_ge_u32 s0, s4
-; GFX9-NEXT: s_cselect_b32 s13, 1, 0
-; GFX9-NEXT: s_add_i32 s15, s12, 1
-; GFX9-NEXT: s_cmp_lg_u32 s13, 0
-; GFX9-NEXT: s_cselect_b32 s12, s15, s12
-; GFX9-NEXT: s_sub_i32 s15, s0, s4
-; GFX9-NEXT: s_cmp_lg_u32 s13, 0
-; GFX9-NEXT: s_cselect_b32 s13, s15, s0
-; GFX9-NEXT: s_cmp_ge_u32 s13, s4
-; GFX9-NEXT: s_cselect_b32 s15, 1, 0
-; GFX9-NEXT: s_add_i32 s0, s12, 1
-; GFX9-NEXT: s_cmp_lg_u32 s15, 0
-; GFX9-NEXT: s_cselect_b32 s0, s0, s12
-; GFX9-NEXT: s_sub_i32 s4, s13, s4
-; GFX9-NEXT: s_cmp_lg_u32 s15, 0
-; GFX9-NEXT: v_readfirstlane_b32 s14, v1
-; GFX9-NEXT: s_cselect_b32 s4, s4, s13
-; GFX9-NEXT: s_sub_i32 s12, 0, s5
-; GFX9-NEXT: s_mul_i32 s12, s12, s14
-; GFX9-NEXT: s_mul_hi_u32 s12, s14, s12
-; GFX9-NEXT: s_add_i32 s14, s14, s12
-; GFX9-NEXT: s_mul_hi_u32 s12, s1, s14
-; GFX9-NEXT: s_mul_i32 s13, s12, s5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX9-NEXT: s_sub_i32 s1, s1, s13
-; GFX9-NEXT: s_cmp_ge_u32 s1, s5
-; GFX9-NEXT: s_cselect_b32 s13, 1, 0
-; GFX9-NEXT: s_add_i32 s14, s12, 1
-; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: v_readfirstlane_b32 s9, v0
+; GFX9-NEXT: s_mul_i32 s8, s8, s9
+; GFX9-NEXT: s_mul_hi_u32 s8, s9, s8
+; GFX9-NEXT: s_add_i32 s9, s9, s8
+; GFX9-NEXT: s_mul_hi_u32 s8, s4, s9
+; GFX9-NEXT: s_mul_i32 s9, s8, s12
+; GFX9-NEXT: s_sub_i32 s4, s4, s9
+; GFX9-NEXT: s_cmp_ge_u32 s4, s12
+; GFX9-NEXT: s_cselect_b32 s9, 1, 0
+; GFX9-NEXT: s_add_i32 s11, s8, 1
+; GFX9-NEXT: s_cmp_lg_u32 s9, 0
+; GFX9-NEXT: s_cselect_b32 s8, s11, s8
+; GFX9-NEXT: s_sub_i32 s11, s4, s12
+; GFX9-NEXT: s_cmp_lg_u32 s9, 0
+; GFX9-NEXT: s_cselect_b32 s9, s11, s4
+; GFX9-NEXT: s_cmp_ge_u32 s9, s12
+; GFX9-NEXT: s_cselect_b32 s11, 1, 0
+; GFX9-NEXT: s_add_i32 s4, s8, 1
+; GFX9-NEXT: s_cmp_lg_u32 s11, 0
+; GFX9-NEXT: s_cselect_b32 s4, s4, s8
+; GFX9-NEXT: s_sub_i32 s8, s9, s12
+; GFX9-NEXT: s_cmp_lg_u32 s11, 0
+; GFX9-NEXT: v_readfirstlane_b32 s10, v1
+; GFX9-NEXT: s_cselect_b32 s8, s8, s9
+; GFX9-NEXT: s_sub_i32 s9, 0, s13
+; GFX9-NEXT: s_mul_i32 s9, s9, s10
+; GFX9-NEXT: s_mul_hi_u32 s9, s10, s9
+; GFX9-NEXT: s_add_i32 s10, s10, s9
+; GFX9-NEXT: s_mul_hi_u32 s9, s5, s10
+; GFX9-NEXT: s_mul_i32 s10, s9, s13
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s14
+; GFX9-NEXT: s_sub_i32 s5, s5, s10
+; GFX9-NEXT: s_cmp_ge_u32 s5, s13
+; GFX9-NEXT: s_cselect_b32 s10, 1, 0
+; GFX9-NEXT: s_add_i32 s11, s9, 1
+; GFX9-NEXT: s_cmp_lg_u32 s10, 0
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT: s_cselect_b32 s12, s14, s12
-; GFX9-NEXT: s_sub_i32 s14, s1, s5
-; GFX9-NEXT: s_cmp_lg_u32 s13, 0
-; GFX9-NEXT: s_cselect_b32 s13, s14, s1
-; GFX9-NEXT: s_cmp_ge_u32 s13, s5
+; GFX9-NEXT: s_cselect_b32 s9, s11, s9
+; GFX9-NEXT: s_sub_i32 s11, s5, s13
+; GFX9-NEXT: s_cmp_lg_u32 s10, 0
+; GFX9-NEXT: s_cselect_b32 s10, s11, s5
+; GFX9-NEXT: s_cmp_ge_u32 s10, s13
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX9-NEXT: s_cselect_b32 s14, 1, 0
-; GFX9-NEXT: s_add_i32 s1, s12, 1
+; GFX9-NEXT: s_cselect_b32 s11, 1, 0
+; GFX9-NEXT: s_add_i32 s5, s9, 1
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: s_cmp_lg_u32 s14, 0
-; GFX9-NEXT: s_cselect_b32 s1, s1, s12
-; GFX9-NEXT: s_sub_i32 s5, s13, s5
-; GFX9-NEXT: s_cmp_lg_u32 s14, 0
-; GFX9-NEXT: s_cselect_b32 s5, s5, s13
-; GFX9-NEXT: v_readfirstlane_b32 s12, v0
-; GFX9-NEXT: s_sub_i32 s13, 0, s6
-; GFX9-NEXT: s_mul_i32 s13, s13, s12
-; GFX9-NEXT: s_mul_hi_u32 s13, s12, s13
-; GFX9-NEXT: s_add_i32 s12, s12, s13
-; GFX9-NEXT: s_mul_hi_u32 s12, s2, s12
-; GFX9-NEXT: s_mul_i32 s13, s12, s6
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_sub_i32 s2, s2, s13
-; GFX9-NEXT: s_cmp_ge_u32 s2, s6
-; GFX9-NEXT: s_cselect_b32 s13, 1, 0
-; GFX9-NEXT: s_add_i32 s14, s12, 1
-; GFX9-NEXT: s_cmp_lg_u32 s13, 0
+; GFX9-NEXT: s_cmp_lg_u32 s11, 0
+; GFX9-NEXT: s_cselect_b32 s5, s5, s9
+; GFX9-NEXT: s_sub_i32 s9, s10, s13
+; GFX9-NEXT: s_cmp_lg_u32 s11, 0
+; GFX9-NEXT: s_cselect_b32 s9, s9, s10
+; GFX9-NEXT: v_readfirstlane_b32 s10, v0
+; GFX9-NEXT: s_sub_i32 s11, 0, s14
+; GFX9-NEXT: s_mul_i32 s11, s11, s10
+; GFX9-NEXT: s_mul_hi_u32 s11, s10, s11
+; GFX9-NEXT: s_add_i32 s10, s10, s11
+; GFX9-NEXT: s_mul_hi_u32 s10, s6, s10
+; GFX9-NEXT: s_mul_i32 s11, s10, s14
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s15
+; GFX9-NEXT: s_sub_i32 s6, s6, s11
+; GFX9-NEXT: s_cmp_ge_u32 s6, s14
+; GFX9-NEXT: s_cselect_b32 s11, 1, 0
+; GFX9-NEXT: s_add_i32 s12, s10, 1
+; GFX9-NEXT: s_cmp_lg_u32 s11, 0
; GFX9-NEXT: v_rcp_iflag_f32_e32 v0, v0
-; GFX9-NEXT: s_cselect_b32 s12, s14, s12
-; GFX9-NEXT: s_sub_i32 s14, s2, s6
-; GFX9-NEXT: s_cmp_lg_u32 s13, 0
-; GFX9-NEXT: s_cselect_b32 s13, s14, s2
-; GFX9-NEXT: s_cmp_ge_u32 s13, s6
+; GFX9-NEXT: s_cselect_b32 s10, s12, s10
+; GFX9-NEXT: s_sub_i32 s12, s6, s14
+; GFX9-NEXT: s_cmp_lg_u32 s11, 0
+; GFX9-NEXT: s_cselect_b32 s11, s12, s6
+; GFX9-NEXT: s_cmp_ge_u32 s11, s14
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX9-NEXT: s_cselect_b32 s14, 1, 0
-; GFX9-NEXT: s_add_i32 s2, s12, 1
+; GFX9-NEXT: s_cselect_b32 s12, 1, 0
+; GFX9-NEXT: s_add_i32 s6, s10, 1
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: s_cmp_lg_u32 s14, 0
-; GFX9-NEXT: s_cselect_b32 s2, s2, s12
-; GFX9-NEXT: s_sub_i32 s6, s13, s6
-; GFX9-NEXT: s_cmp_lg_u32 s14, 0
-; GFX9-NEXT: s_cselect_b32 s6, s6, s13
-; GFX9-NEXT: v_readfirstlane_b32 s12, v0
-; GFX9-NEXT: s_sub_i32 s13, 0, s7
-; GFX9-NEXT: s_mul_i32 s13, s13, s12
-; GFX9-NEXT: s_mul_hi_u32 s13, s12, s13
-; GFX9-NEXT: s_add_i32 s12, s12, s13
-; GFX9-NEXT: s_mul_hi_u32 s12, s3, s12
-; GFX9-NEXT: s_mul_i32 s13, s12, s7
-; GFX9-NEXT: s_sub_i32 s3, s3, s13
-; GFX9-NEXT: s_cmp_ge_u32 s3, s7
+; GFX9-NEXT: s_cmp_lg_u32 s12, 0
+; GFX9-NEXT: s_cselect_b32 s6, s6, s10
+; GFX9-NEXT: s_sub_i32 s10, s11, s14
+; GFX9-NEXT: s_cmp_lg_u32 s12, 0
+; GFX9-NEXT: s_cselect_b32 s10, s10, s11
+; GFX9-NEXT: v_readfirstlane_b32 s11, v0
+; GFX9-NEXT: s_sub_i32 s12, 0, s15
+; GFX9-NEXT: s_mul_i32 s12, s12, s11
+; GFX9-NEXT: s_mul_hi_u32 s12, s11, s12
+; GFX9-NEXT: s_add_i32 s11, s11, s12
+; GFX9-NEXT: s_mul_hi_u32 s11, s7, s11
+; GFX9-NEXT: s_mul_i32 s12, s11, s15
+; GFX9-NEXT: s_sub_i32 s7, s7, s12
+; GFX9-NEXT: s_cmp_ge_u32 s7, s15
+; GFX9-NEXT: s_cselect_b32 s12, 1, 0
+; GFX9-NEXT: s_add_i32 s13, s11, 1
+; GFX9-NEXT: s_cmp_lg_u32 s12, 0
+; GFX9-NEXT: s_cselect_b32 s11, s13, s11
+; GFX9-NEXT: s_sub_i32 s13, s7, s15
+; GFX9-NEXT: s_cmp_lg_u32 s12, 0
+; GFX9-NEXT: s_cselect_b32 s12, s13, s7
+; GFX9-NEXT: s_cmp_ge_u32 s12, s15
; GFX9-NEXT: s_cselect_b32 s13, 1, 0
-; GFX9-NEXT: s_add_i32 s14, s12, 1
+; GFX9-NEXT: s_add_i32 s7, s11, 1
; GFX9-NEXT: s_cmp_lg_u32 s13, 0
-; GFX9-NEXT: s_cselect_b32 s12, s14, s12
-; GFX9-NEXT: s_sub_i32 s14, s3, s7
-; GFX9-NEXT: s_cmp_lg_u32 s13, 0
-; GFX9-NEXT: s_cselect_b32 s13, s14, s3
-; GFX9-NEXT: s_cmp_ge_u32 s13, s7
-; GFX9-NEXT: s_cselect_b32 s14, 1, 0
-; GFX9-NEXT: s_add_i32 s3, s12, 1
-; GFX9-NEXT: s_cmp_lg_u32 s14, 0
-; GFX9-NEXT: s_cselect_b32 s3, s3, s12
-; GFX9-NEXT: s_sub_i32 s7, s13, s7
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: s_cmp_lg_u32 s14, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: s_cselect_b32 s7, s7, s13
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9]
-; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: s_cselect_b32 s7, s7, s11
+; GFX9-NEXT: s_sub_i32 s11, s12, s15
; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: s_cmp_lg_u32 s13, 0
; GFX9-NEXT: v_mov_b32_e32 v1, s5
; GFX9-NEXT: v_mov_b32_e32 v2, s6
; GFX9-NEXT: v_mov_b32_e32 v3, s7
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[10:11]
+; GFX9-NEXT: s_cselect_b32 s11, s11, s12
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[0:1]
+; GFX9-NEXT: s_nop 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s8
+; GFX9-NEXT: v_mov_b32_e32 v1, s9
+; GFX9-NEXT: v_mov_b32_e32 v2, s10
+; GFX9-NEXT: v_mov_b32_e32 v3, s11
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: udivrem_v4i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x10
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[12:15], s[8:9], 0x20
+; GFX10-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v8, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s4
-; GFX10-NEXT: s_sub_i32 s11, 0, s4
-; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s5
+; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s12
+; GFX10-NEXT: s_sub_i32 s9, 0, s12
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s13
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s10, v0
+; GFX10-NEXT: v_readfirstlane_b32 s8, v0
; GFX10-NEXT: v_rcp_iflag_f32_e32 v0, v1
-; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s6
-; GFX10-NEXT: s_mul_i32 s11, s11, s10
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s14
+; GFX10-NEXT: s_mul_i32 s9, s9, s8
; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GFX10-NEXT: s_mul_hi_u32 s11, s10, s11
-; GFX10-NEXT: s_add_i32 s10, s10, s11
+; GFX10-NEXT: s_mul_hi_u32 s9, s8, s9
+; GFX10-NEXT: s_add_i32 s8, s8, s9
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
-; GFX10-NEXT: s_mul_hi_u32 s10, s0, s10
-; GFX10-NEXT: s_mul_i32 s11, s10, s4
-; GFX10-NEXT: s_sub_i32 s0, s0, s11
+; GFX10-NEXT: s_mul_hi_u32 s8, s4, s8
+; GFX10-NEXT: s_mul_i32 s9, s8, s12
+; GFX10-NEXT: s_sub_i32 s4, s4, s9
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: s_cmp_ge_u32 s0, s4
-; GFX10-NEXT: s_cselect_b32 s11, 1, 0
-; GFX10-NEXT: s_add_i32 s12, s10, 1
-; GFX10-NEXT: s_cmp_lg_u32 s11, 0
-; GFX10-NEXT: v_readfirstlane_b32 s13, v0
-; GFX10-NEXT: s_cselect_b32 s10, s12, s10
-; GFX10-NEXT: s_sub_i32 s12, s0, s4
-; GFX10-NEXT: s_cmp_lg_u32 s11, 0
+; GFX10-NEXT: s_cmp_ge_u32 s4, s12
+; GFX10-NEXT: s_cselect_b32 s9, 1, 0
+; GFX10-NEXT: s_add_i32 s10, s8, 1
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
+; GFX10-NEXT: v_readfirstlane_b32 s11, v0
+; GFX10-NEXT: s_cselect_b32 s8, s10, s8
+; GFX10-NEXT: s_sub_i32 s10, s4, s12
+; GFX10-NEXT: s_cmp_lg_u32 s9, 0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v1
-; GFX10-NEXT: s_cselect_b32 s11, s12, s0
-; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s7
-; GFX10-NEXT: s_cmp_ge_u32 s11, s4
-; GFX10-NEXT: s_cselect_b32 s12, 1, 0
-; GFX10-NEXT: s_add_i32 s0, s10, 1
-; GFX10-NEXT: s_cmp_lg_u32 s12, 0
+; GFX10-NEXT: s_cselect_b32 s9, s10, s4
+; GFX10-NEXT: v_cvt_f32_u32_e32 v1, s15
+; GFX10-NEXT: s_cmp_ge_u32 s9, s12
+; GFX10-NEXT: s_cselect_b32 s10, 1, 0
+; GFX10-NEXT: s_add_i32 s4, s8, 1
+; GFX10-NEXT: s_cmp_lg_u32 s10, 0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: s_cselect_b32 s0, s0, s10
-; GFX10-NEXT: s_sub_i32 s4, s11, s4
-; GFX10-NEXT: s_cmp_lg_u32 s12, 0
+; GFX10-NEXT: s_cselect_b32 s4, s4, s8
+; GFX10-NEXT: s_sub_i32 s8, s9, s12
+; GFX10-NEXT: s_cmp_lg_u32 s10, 0
; GFX10-NEXT: v_rcp_iflag_f32_e32 v1, v1
-; GFX10-NEXT: s_cselect_b32 s4, s4, s11
-; GFX10-NEXT: s_sub_i32 s10, 0, s5
-; GFX10-NEXT: s_mul_i32 s10, s10, s13
-; GFX10-NEXT: s_mul_hi_u32 s10, s13, s10
-; GFX10-NEXT: s_add_i32 s13, s13, s10
-; GFX10-NEXT: s_mul_hi_u32 s10, s1, s13
-; GFX10-NEXT: v_readfirstlane_b32 s13, v0
-; GFX10-NEXT: s_mul_i32 s11, s10, s5
+; GFX10-NEXT: s_cselect_b32 s8, s8, s9
+; GFX10-NEXT: s_sub_i32 s9, 0, s13
+; GFX10-NEXT: v_readfirstlane_b32 s12, v0
+; GFX10-NEXT: s_mul_i32 s9, s9, s11
+; GFX10-NEXT: s_mul_hi_u32 s9, s11, s9
+; GFX10-NEXT: s_add_i32 s11, s11, s9
+; GFX10-NEXT: s_mul_hi_u32 s9, s5, s11
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v1
-; GFX10-NEXT: s_sub_i32 s1, s1, s11
-; GFX10-NEXT: s_cmp_ge_u32 s1, s5
+; GFX10-NEXT: s_mul_i32 s10, s9, s13
+; GFX10-NEXT: s_sub_i32 s5, s5, s10
+; GFX10-NEXT: s_cmp_ge_u32 s5, s13
+; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
+; GFX10-NEXT: s_cselect_b32 s10, 1, 0
+; GFX10-NEXT: s_add_i32 s11, s9, 1
+; GFX10-NEXT: s_cmp_lg_u32 s10, 0
+; GFX10-NEXT: s_cselect_b32 s9, s11, s9
+; GFX10-NEXT: s_sub_i32 s11, s5, s13
+; GFX10-NEXT: s_cmp_lg_u32 s10, 0
+; GFX10-NEXT: s_cselect_b32 s10, s11, s5
+; GFX10-NEXT: s_cmp_ge_u32 s10, s13
; GFX10-NEXT: s_cselect_b32 s11, 1, 0
-; GFX10-NEXT: s_add_i32 s12, s10, 1
+; GFX10-NEXT: s_add_i32 s5, s9, 1
; GFX10-NEXT: s_cmp_lg_u32 s11, 0
-; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: s_cselect_b32 s10, s12, s10
-; GFX10-NEXT: s_sub_i32 s12, s1, s5
+; GFX10-NEXT: s_cselect_b32 s5, s5, s9
+; GFX10-NEXT: s_sub_i32 s9, s10, s13
; GFX10-NEXT: s_cmp_lg_u32 s11, 0
-; GFX10-NEXT: s_cselect_b32 s11, s12, s1
-; GFX10-NEXT: s_cmp_ge_u32 s11, s5
-; GFX10-NEXT: s_cselect_b32 s12, 1, 0
-; GFX10-NEXT: s_add_i32 s1, s10, 1
-; GFX10-NEXT: s_cmp_lg_u32 s12, 0
-; GFX10-NEXT: s_cselect_b32 s1, s1, s10
-; GFX10-NEXT: s_sub_i32 s5, s11, s5
-; GFX10-NEXT: s_cmp_lg_u32 s12, 0
-; GFX10-NEXT: s_cselect_b32 s5, s5, s11
-; GFX10-NEXT: s_sub_i32 s10, 0, s6
-; GFX10-NEXT: s_mul_i32 s10, s10, s13
-; GFX10-NEXT: s_mul_hi_u32 s10, s13, s10
-; GFX10-NEXT: s_add_i32 s13, s13, s10
-; GFX10-NEXT: s_mul_hi_u32 s10, s2, s13
; GFX10-NEXT: v_readfirstlane_b32 s13, v0
-; GFX10-NEXT: s_mul_i32 s11, s10, s6
-; GFX10-NEXT: s_sub_i32 s2, s2, s11
-; GFX10-NEXT: s_cmp_ge_u32 s2, s6
+; GFX10-NEXT: s_cselect_b32 s9, s9, s10
+; GFX10-NEXT: s_sub_i32 s10, 0, s14
+; GFX10-NEXT: s_mul_i32 s10, s10, s12
+; GFX10-NEXT: s_mul_hi_u32 s10, s12, s10
+; GFX10-NEXT: s_add_i32 s12, s12, s10
+; GFX10-NEXT: s_mul_hi_u32 s10, s6, s12
+; GFX10-NEXT: s_mul_i32 s11, s10, s14
+; GFX10-NEXT: s_sub_i32 s6, s6, s11
+; GFX10-NEXT: s_cmp_ge_u32 s6, s14
; GFX10-NEXT: s_cselect_b32 s11, 1, 0
; GFX10-NEXT: s_add_i32 s12, s10, 1
; GFX10-NEXT: s_cmp_lg_u32 s11, 0
; GFX10-NEXT: s_cselect_b32 s10, s12, s10
-; GFX10-NEXT: s_sub_i32 s12, s2, s6
+; GFX10-NEXT: s_sub_i32 s12, s6, s14
; GFX10-NEXT: s_cmp_lg_u32 s11, 0
-; GFX10-NEXT: s_cselect_b32 s11, s12, s2
-; GFX10-NEXT: s_cmp_ge_u32 s11, s6
+; GFX10-NEXT: s_cselect_b32 s11, s12, s6
+; GFX10-NEXT: s_cmp_ge_u32 s11, s14
; GFX10-NEXT: s_cselect_b32 s12, 1, 0
-; GFX10-NEXT: s_add_i32 s2, s10, 1
+; GFX10-NEXT: s_add_i32 s6, s10, 1
; GFX10-NEXT: s_cmp_lg_u32 s12, 0
-; GFX10-NEXT: s_cselect_b32 s2, s2, s10
-; GFX10-NEXT: s_sub_i32 s6, s11, s6
+; GFX10-NEXT: s_cselect_b32 s6, s6, s10
+; GFX10-NEXT: s_sub_i32 s10, s11, s14
; GFX10-NEXT: s_cmp_lg_u32 s12, 0
-; GFX10-NEXT: s_cselect_b32 s6, s6, s11
-; GFX10-NEXT: s_sub_i32 s10, 0, s7
-; GFX10-NEXT: s_mul_i32 s10, s10, s13
-; GFX10-NEXT: s_mul_hi_u32 s10, s13, s10
-; GFX10-NEXT: s_add_i32 s13, s13, s10
-; GFX10-NEXT: s_mul_hi_u32 s12, s3, s13
-; GFX10-NEXT: s_mul_i32 s10, s12, s7
-; GFX10-NEXT: s_sub_i32 s3, s3, s10
-; GFX10-NEXT: s_load_dwordx4 s[8:11], s[8:9], 0x0
-; GFX10-NEXT: s_cmp_ge_u32 s3, s7
+; GFX10-NEXT: s_cselect_b32 s10, s10, s11
+; GFX10-NEXT: s_sub_i32 s11, 0, s15
+; GFX10-NEXT: s_mul_i32 s11, s11, s13
+; GFX10-NEXT: s_mul_hi_u32 s11, s13, s11
+; GFX10-NEXT: s_add_i32 s13, s13, s11
+; GFX10-NEXT: s_mul_hi_u32 s11, s7, s13
+; GFX10-NEXT: s_mul_i32 s12, s11, s15
+; GFX10-NEXT: s_sub_i32 s7, s7, s12
+; GFX10-NEXT: s_cmp_ge_u32 s7, s15
+; GFX10-NEXT: s_cselect_b32 s12, 1, 0
+; GFX10-NEXT: s_add_i32 s13, s11, 1
+; GFX10-NEXT: s_cmp_lg_u32 s12, 0
+; GFX10-NEXT: s_cselect_b32 s11, s13, s11
+; GFX10-NEXT: s_sub_i32 s13, s7, s15
+; GFX10-NEXT: s_cmp_lg_u32 s12, 0
+; GFX10-NEXT: s_cselect_b32 s12, s13, s7
+; GFX10-NEXT: s_cmp_ge_u32 s12, s15
; GFX10-NEXT: s_cselect_b32 s13, 1, 0
-; GFX10-NEXT: s_add_i32 s14, s12, 1
+; GFX10-NEXT: s_add_i32 s7, s11, 1
; GFX10-NEXT: s_cmp_lg_u32 s13, 0
-; GFX10-NEXT: s_cselect_b32 s12, s14, s12
-; GFX10-NEXT: s_sub_i32 s14, s3, s7
+; GFX10-NEXT: s_cselect_b32 s7, s7, s11
+; GFX10-NEXT: s_sub_i32 s11, s12, s15
; GFX10-NEXT: s_cmp_lg_u32 s13, 0
-; GFX10-NEXT: s_cselect_b32 s13, s14, s3
-; GFX10-NEXT: s_cmp_ge_u32 s13, s7
-; GFX10-NEXT: s_cselect_b32 s14, 1, 0
-; GFX10-NEXT: s_add_i32 s3, s12, 1
-; GFX10-NEXT: s_cmp_lg_u32 s14, 0
-; GFX10-NEXT: s_cselect_b32 s3, s3, s12
-; GFX10-NEXT: s_sub_i32 s7, s13, s7
-; GFX10-NEXT: s_cmp_lg_u32 s14, 0
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_cselect_b32 s7, s7, s13
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: v_mov_b32_e32 v3, s3
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
-; GFX10-NEXT: v_mov_b32_e32 v5, s5
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v7, s7
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[8:9]
-; GFX10-NEXT: global_store_dwordx4 v8, v[4:7], s[10:11]
+; GFX10-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-NEXT: s_cselect_b32 s11, s11, s12
+; GFX10-NEXT: v_mov_b32_e32 v1, s5
+; GFX10-NEXT: v_mov_b32_e32 v2, s6
+; GFX10-NEXT: v_mov_b32_e32 v3, s7
+; GFX10-NEXT: v_mov_b32_e32 v4, s8
+; GFX10-NEXT: v_mov_b32_e32 v5, s9
+; GFX10-NEXT: v_mov_b32_e32 v6, s10
+; GFX10-NEXT: v_mov_b32_e32 v7, s11
+; GFX10-NEXT: global_store_dwordx4 v8, v[0:3], s[0:1]
+; GFX10-NEXT: global_store_dwordx4 v8, v[4:7], s[2:3]
; GFX10-NEXT: s_endpgm
%div = udiv <4 x i32> %x, %y
store <4 x i32> %div, ptr addrspace(1) %out0
diff --git a/llvm/test/CodeGen/AMDGPU/abs_i32.ll b/llvm/test/CodeGen/AMDGPU/abs_i32.ll
index b53047fadf382..4802dc872f4a2 100644
--- a/llvm/test/CodeGen/AMDGPU/abs_i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/abs_i32.ll
@@ -5,8 +5,7 @@
define amdgpu_kernel void @abs_v1(ptr addrspace(1) %out, i32 %arg) {
; GFX9-LABEL: abs_v1:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_abs_i32 s2, s2
@@ -34,8 +33,7 @@ define amdgpu_kernel void @abs_v1(ptr addrspace(1) %out, i32 %arg) {
define amdgpu_kernel void @abs_v2(ptr addrspace(1) %out, i32 %arg) {
; GFX9-LABEL: abs_v2:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_abs_i32 s2, s2
@@ -64,8 +62,7 @@ define amdgpu_kernel void @abs_v2(ptr addrspace(1) %out, i32 %arg) {
define amdgpu_kernel void @abs_v3(ptr addrspace(1) %out, i32 %arg) {
; GFX9-LABEL: abs_v3:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_abs_i32 s2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
index 96906ca0a5bbf..33d459fdd182d 100644
--- a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
@@ -224,16 +224,16 @@ define amdgpu_kernel void @s_test_add_v2i16_kernarg(ptr addrspace(1) %out, <2 x
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s4, s3, 16
-; VI-NEXT: s_lshr_b32 s5, s2, 16
-; VI-NEXT: s_add_i32 s2, s2, s3
-; VI-NEXT: s_add_i32 s5, s5, s4
-; VI-NEXT: s_and_b32 s2, s2, 0xffff
-; VI-NEXT: s_lshl_b32 s3, s5, 16
-; VI-NEXT: s_or_b32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_lshr_b32 s0, s3, 16
+; VI-NEXT: s_lshr_b32 s1, s2, 16
+; VI-NEXT: s_add_i32 s2, s2, s3
+; VI-NEXT: s_add_i32 s1, s1, s0
+; VI-NEXT: s_lshl_b32 s0, s1, 16
+; VI-NEXT: s_and_b32 s1, s2, 0xffff
+; VI-NEXT: s_or_b32 s0, s1, s0
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/add_i128.ll b/llvm/test/CodeGen/AMDGPU/add_i128.ll
index dcaa856cc1016..5a3c378b901d0 100644
--- a/llvm/test/CodeGen/AMDGPU/add_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/add_i128.ll
@@ -90,15 +90,17 @@ define amdgpu_kernel void @sgpr_operand_reversed(ptr addrspace(1) noalias %out,
define amdgpu_kernel void @test_sreg(ptr addrspace(1) noalias %out, i128 %a, i128 %b) {
; GCN-LABEL: test_sreg:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x11
; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_add_u32 s4, s8, s12
-; GCN-NEXT: s_addc_u32 s5, s9, s13
-; GCN-NEXT: s_addc_u32 s6, s10, s14
-; GCN-NEXT: s_addc_u32 s7, s11, s15
+; GCN-NEXT: s_add_u32 s4, s10, s14
+; GCN-NEXT: s_addc_u32 s5, s11, s15
+; GCN-NEXT: s_addc_u32 s6, s12, s0
+; GCN-NEXT: s_addc_u32 s7, s13, s1
; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_mov_b32 s0, s8
+; GCN-NEXT: s_mov_b32 s1, s9
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: v_mov_b32_e32 v2, s6
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
index 4652d0ae60fd0..6e530bdbb9ea6 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
@@ -364,12 +364,13 @@ define amdgpu_kernel void @store_flat_scratch(ptr addrspace(1) noalias %out, i32
}
; HSA-LABEL: {{^}}use_constant_to_constant32_addrspacecast
-; GFX9: s_load_dwordx2 [[PTRPTR:s\[[0-9]+:[0-9]+\]]], s[4:5], 0x0{{$}}
-; GFX9: s_load_dword [[OFFSET:s[0-9]+]], s[4:5], 0x8{{$}}
-; GFX9: s_load_dwordx2 s[[[PTR_LO:[0-9]+]]:[[PTR_HI:[0-9]+]]], [[PTRPTR]], 0x0{{$}}
-; GFX9: s_mov_b32 s[[PTR_HI]], 0{{$}}
-; GFX9: s_add_i32 s[[PTR_LO]], s[[PTR_LO]], [[OFFSET]]
-; GFX9: s_load_dword s{{[0-9]+}}, s[[[PTR_LO]]:[[PTR_HI]]], 0x0{{$}}
+; GFX9: s_load_dwordx4 s[0:3], s[4:5], 0x0{{$}}
+; GFX9: s_waitcnt lgkmcnt(0){{$}}
+; GFX9: s_load_dwordx2 s[0:1], s[0:1], 0x0{{$}}
+; GFX9: s_waitcnt lgkmcnt(0){{$}}
+; GFX9: s_mov_b32 s1, 0{{$}}
+; GFX9: s_add_i32 s0, s0, s2{{$}}
+; GFX9: s_load_dword s0, s[0:1], 0x0{{$}}
define amdgpu_kernel void @use_constant_to_constant32_addrspacecast(ptr addrspace(4) %ptr.ptr, i32 %offset) #0 {
%ptr = load volatile ptr addrspace(4), ptr addrspace(4) %ptr.ptr
%addrspacecast = addrspacecast ptr addrspace(4) %ptr to ptr addrspace(6)
@@ -379,12 +380,13 @@ define amdgpu_kernel void @use_constant_to_constant32_addrspacecast(ptr addrspac
}
; HSA-LABEL: {{^}}use_global_to_constant32_addrspacecast
-; GFX9: s_load_dwordx2 [[PTRPTR:s\[[0-9]+:[0-9]+\]]], s[4:5], 0x0{{$}}
-; GFX9: s_load_dword [[OFFSET:s[0-9]+]], s[4:5], 0x8{{$}}
-; GFX9: s_load_dwordx2 s[[[PTR_LO:[0-9]+]]:[[PTR_HI:[0-9]+]]], [[PTRPTR]], 0x0{{$}}
-; GFX9: s_mov_b32 s[[PTR_HI]], 0{{$}}
-; GFX9: s_add_i32 s[[PTR_LO]], s[[PTR_LO]], [[OFFSET]]
-; GFX9: s_load_dword s{{[0-9]+}}, s[[[PTR_LO]]:[[PTR_HI]]], 0x0{{$}}
+; GFX9: s_load_dwordx4 s[0:3], s[4:5], 0x0{{$}}
+; GFX9: s_waitcnt lgkmcnt(0){{$}}
+; GFX9: s_load_dwordx2 s[0:1], s[0:1], 0x0{{$}}
+; GFX9: s_waitcnt lgkmcnt(0){{$}}
+; GFX9: s_mov_b32 s1, 0{{$}}
+; GFX9: s_add_i32 s0, s0, s2{{$}}
+; GFX9: s_load_dword s0, s[0:1], 0x0{{$}}
define amdgpu_kernel void @use_global_to_constant32_addrspacecast(ptr addrspace(4) %ptr.ptr, i32 %offset) #0 {
%ptr = load volatile ptr addrspace(1), ptr addrspace(4) %ptr.ptr
%addrspacecast = addrspacecast ptr addrspace(1) %ptr to ptr addrspace(6)
diff --git a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
index 727728145dfe0..10cb12407b177 100644
--- a/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
+++ b/llvm/test/CodeGen/AMDGPU/agpr-copy-no-free-registers.ll
@@ -511,52 +511,50 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-LABEL: introduced_copy_to_sgpr:
; GFX908: ; %bb.0: ; %bb
; GFX908-NEXT: global_load_ushort v16, v[0:1], off glc
-; GFX908-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
-; GFX908-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x10
-; GFX908-NEXT: s_load_dword s0, s[8:9], 0x18
-; GFX908-NEXT: s_mov_b32 s12, 0
-; GFX908-NEXT: s_mov_b32 s9, s12
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX908-NEXT: s_sub_i32 s1, 0, s7
-; GFX908-NEXT: v_cvt_f32_f16_e32 v18, s0
+; GFX908-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x0
+; GFX908-NEXT: s_mov_b32 s4, 0
+; GFX908-NEXT: s_mov_b32 s7, s4
; GFX908-NEXT: v_mov_b32_e32 v17, 0
-; GFX908-NEXT: v_rcp_f32_e32 v0, v0
; GFX908-NEXT: v_mov_b32_e32 v1, 0
+; GFX908-NEXT: s_waitcnt lgkmcnt(0)
+; GFX908-NEXT: v_cvt_f32_u32_e32 v0, s15
+; GFX908-NEXT: s_sub_i32 s0, 0, s15
+; GFX908-NEXT: v_cvt_f32_f16_e32 v18, s18
+; GFX908-NEXT: v_rcp_f32_e32 v0, v0
; GFX908-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX908-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX908-NEXT: v_readfirstlane_b32 s2, v0
-; GFX908-NEXT: s_mul_i32 s1, s1, s2
-; GFX908-NEXT: s_mul_hi_u32 s1, s2, s1
-; GFX908-NEXT: s_add_i32 s2, s2, s1
-; GFX908-NEXT: s_mul_hi_u32 s1, s6, s2
-; GFX908-NEXT: s_mul_i32 s2, s1, s7
-; GFX908-NEXT: s_sub_i32 s2, s6, s2
-; GFX908-NEXT: s_add_i32 s3, s1, 1
-; GFX908-NEXT: s_sub_i32 s6, s2, s7
-; GFX908-NEXT: s_cmp_ge_u32 s2, s7
+; GFX908-NEXT: v_readfirstlane_b32 s1, v0
+; GFX908-NEXT: s_mul_i32 s0, s0, s1
+; GFX908-NEXT: s_mul_hi_u32 s0, s1, s0
+; GFX908-NEXT: s_add_i32 s1, s1, s0
+; GFX908-NEXT: s_mul_hi_u32 s0, s14, s1
+; GFX908-NEXT: s_mul_i32 s1, s0, s15
+; GFX908-NEXT: s_sub_i32 s1, s14, s1
+; GFX908-NEXT: s_add_i32 s2, s0, 1
+; GFX908-NEXT: s_sub_i32 s3, s1, s15
+; GFX908-NEXT: s_cmp_ge_u32 s1, s15
+; GFX908-NEXT: s_cselect_b32 s0, s2, s0
; GFX908-NEXT: s_cselect_b32 s1, s3, s1
-; GFX908-NEXT: s_cselect_b32 s2, s6, s2
-; GFX908-NEXT: s_add_i32 s3, s1, 1
-; GFX908-NEXT: s_cmp_ge_u32 s2, s7
-; GFX908-NEXT: s_cselect_b32 s8, s3, s1
-; GFX908-NEXT: s_lshr_b32 s2, s0, 16
+; GFX908-NEXT: s_add_i32 s2, s0, 1
+; GFX908-NEXT: s_cmp_ge_u32 s1, s15
+; GFX908-NEXT: s_cselect_b32 s6, s2, s0
+; GFX908-NEXT: s_lshr_b32 s2, s18, 16
; GFX908-NEXT: v_cvt_f32_f16_e32 v19, s2
-; GFX908-NEXT: s_lshl_b64 s[6:7], s[4:5], 5
-; GFX908-NEXT: s_lshl_b64 s[14:15], s[10:11], 5
+; GFX908-NEXT: s_lshl_b64 s[8:9], s[12:13], 5
+; GFX908-NEXT: s_lshl_b64 s[10:11], s[16:17], 5
; GFX908-NEXT: s_and_b64 s[0:1], exec, s[0:1]
-; GFX908-NEXT: s_lshl_b64 s[16:17], s[8:9], 5
+; GFX908-NEXT: s_lshl_b64 s[14:15], s[6:7], 5
; GFX908-NEXT: v_mov_b32_e32 v0, 0
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s2, v16
; GFX908-NEXT: s_and_b32 s2, 0xffff, s2
-; GFX908-NEXT: s_mul_i32 s3, s5, s2
-; GFX908-NEXT: s_mul_hi_u32 s5, s4, s2
-; GFX908-NEXT: s_mul_i32 s2, s4, s2
+; GFX908-NEXT: s_mul_i32 s3, s13, s2
+; GFX908-NEXT: s_mul_hi_u32 s5, s12, s2
+; GFX908-NEXT: s_mul_i32 s2, s12, s2
; GFX908-NEXT: s_add_i32 s3, s5, s3
-; GFX908-NEXT: s_lshl_b64 s[4:5], s[2:3], 5
+; GFX908-NEXT: s_lshl_b64 s[12:13], s[2:3], 5
; GFX908-NEXT: s_branch .LBB3_2
-; GFX908-NEXT: .LBB3_1: ; %Flow20
+; GFX908-NEXT: .LBB3_1: ; %Flow23
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX908-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; GFX908-NEXT: s_cbranch_vccz .LBB3_12
@@ -569,40 +567,40 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: ; %bb.3: ; %bb14
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX908-NEXT: global_load_dwordx2 v[2:3], v[0:1], off
-; GFX908-NEXT: s_cmp_lt_i32 s11, 0
-; GFX908-NEXT: s_mov_b32 s13, s12
+; GFX908-NEXT: s_mov_b32 s5, s4
+; GFX908-NEXT: s_cmp_lt_i32 s17, 0
+; GFX908-NEXT: v_mov_b32_e32 v4, s4
+; GFX908-NEXT: v_mov_b32_e32 v7, s5
+; GFX908-NEXT: v_mov_b32_e32 v9, s5
; GFX908-NEXT: s_cselect_b64 s[18:19], -1, 0
-; GFX908-NEXT: s_cmp_gt_i32 s11, -1
-; GFX908-NEXT: v_mov_b32_e32 v4, s12
-; GFX908-NEXT: v_mov_b32_e32 v6, s12
-; GFX908-NEXT: v_mov_b32_e32 v8, s12
-; GFX908-NEXT: v_mov_b32_e32 v5, s13
+; GFX908-NEXT: s_cmp_gt_i32 s17, -1
+; GFX908-NEXT: v_mov_b32_e32 v5, s5
+; GFX908-NEXT: v_mov_b32_e32 v6, s4
+; GFX908-NEXT: v_mov_b32_e32 v8, s4
; GFX908-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX908-NEXT: v_mov_b32_e32 v7, s13
-; GFX908-NEXT: v_mov_b32_e32 v9, s13
; GFX908-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[2:3]
; GFX908-NEXT: v_cmp_ne_u32_e64 s[2:3], 1, v10
; GFX908-NEXT: v_mov_b32_e32 v11, v5
-; GFX908-NEXT: s_mov_b64 s[20:21], s[14:15]
+; GFX908-NEXT: s_mov_b64 s[20:21], s[10:11]
; GFX908-NEXT: v_mov_b32_e32 v10, v4
; GFX908-NEXT: s_waitcnt vmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s9, v2
-; GFX908-NEXT: v_readfirstlane_b32 s13, v3
-; GFX908-NEXT: s_add_u32 s9, s9, 1
-; GFX908-NEXT: s_addc_u32 s13, s13, 0
-; GFX908-NEXT: s_mul_hi_u32 s22, s6, s9
-; GFX908-NEXT: s_mul_i32 s13, s6, s13
-; GFX908-NEXT: s_mul_i32 s23, s7, s9
-; GFX908-NEXT: s_add_i32 s13, s22, s13
-; GFX908-NEXT: s_mul_i32 s9, s6, s9
-; GFX908-NEXT: s_add_i32 s13, s13, s23
+; GFX908-NEXT: v_readfirstlane_b32 s5, v2
+; GFX908-NEXT: v_readfirstlane_b32 s7, v3
+; GFX908-NEXT: s_add_u32 s5, s5, 1
+; GFX908-NEXT: s_addc_u32 s7, s7, 0
+; GFX908-NEXT: s_mul_hi_u32 s22, s8, s5
+; GFX908-NEXT: s_mul_i32 s7, s8, s7
+; GFX908-NEXT: s_mul_i32 s23, s9, s5
+; GFX908-NEXT: s_add_i32 s7, s22, s7
+; GFX908-NEXT: s_mul_i32 s5, s8, s5
+; GFX908-NEXT: s_add_i32 s7, s7, s23
; GFX908-NEXT: s_branch .LBB3_5
; GFX908-NEXT: .LBB3_4: ; %bb58
; GFX908-NEXT: ; in Loop: Header=BB3_5 Depth=2
; GFX908-NEXT: v_add_co_u32_sdwa v2, vcc, v2, v16 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX908-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
-; GFX908-NEXT: s_add_u32 s20, s20, s4
-; GFX908-NEXT: s_addc_u32 s21, s21, s5
+; GFX908-NEXT: s_add_u32 s20, s20, s12
+; GFX908-NEXT: s_addc_u32 s21, s21, s13
; GFX908-NEXT: s_mov_b64 s[22:23], 0
; GFX908-NEXT: v_cmp_lt_i32_e64 s[24:25], -1, v3
; GFX908-NEXT: s_andn2_b64 vcc, exec, s[24:25]
@@ -610,8 +608,8 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: .LBB3_5: ; %bb16
; GFX908-NEXT: ; Parent Loop BB3_2 Depth=1
; GFX908-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX908-NEXT: s_add_u32 s22, s20, s9
-; GFX908-NEXT: s_addc_u32 s23, s21, s13
+; GFX908-NEXT: s_add_u32 s22, s20, s5
+; GFX908-NEXT: s_addc_u32 s23, s21, s7
; GFX908-NEXT: global_load_dword v21, v17, s[22:23] offset:16 glc
; GFX908-NEXT: s_waitcnt vmcnt(0)
; GFX908-NEXT: global_load_dword v20, v17, s[22:23] offset:20 glc
@@ -655,17 +653,17 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX908-NEXT: .LBB3_9: ; %loop.exit.guard
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX908-NEXT: s_xor_b64 s[18:19], s[22:23], -1
-; GFX908-NEXT: .LBB3_10: ; %Flow19
+; GFX908-NEXT: .LBB3_10: ; %Flow22
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX908-NEXT: s_mov_b64 s[2:3], -1
; GFX908-NEXT: s_and_b64 vcc, exec, s[18:19]
; GFX908-NEXT: s_cbranch_vccz .LBB3_1
; GFX908-NEXT: ; %bb.11: ; %bb12
; GFX908-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX908-NEXT: s_add_u32 s10, s10, s8
-; GFX908-NEXT: s_addc_u32 s11, s11, 0
-; GFX908-NEXT: s_add_u32 s14, s14, s16
-; GFX908-NEXT: s_addc_u32 s15, s15, s17
+; GFX908-NEXT: s_add_u32 s16, s16, s6
+; GFX908-NEXT: s_addc_u32 s17, s17, 0
+; GFX908-NEXT: s_add_u32 s10, s10, s14
+; GFX908-NEXT: s_addc_u32 s11, s11, s15
; GFX908-NEXT: s_mov_b64 s[2:3], 0
; GFX908-NEXT: s_branch .LBB3_1
; GFX908-NEXT: .LBB3_12: ; %DummyReturnBlock
@@ -674,51 +672,49 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-LABEL: introduced_copy_to_sgpr:
; GFX90A: ; %bb.0: ; %bb
; GFX90A-NEXT: global_load_ushort v18, v[0:1], off glc
-; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
-; GFX90A-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x10
-; GFX90A-NEXT: s_load_dword s0, s[8:9], 0x18
-; GFX90A-NEXT: s_mov_b32 s12, 0
-; GFX90A-NEXT: s_mov_b32 s9, s12
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX90A-NEXT: s_sub_i32 s1, 0, s7
+; GFX90A-NEXT: s_load_dwordx8 s[12:19], s[8:9], 0x0
+; GFX90A-NEXT: s_mov_b32 s4, 0
+; GFX90A-NEXT: s_mov_b32 s7, s4
; GFX90A-NEXT: v_mov_b32_e32 v19, 0
; GFX90A-NEXT: v_pk_mov_b32 v[2:3], 0, 0
+; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: v_cvt_f32_u32_e32 v0, s15
+; GFX90A-NEXT: s_sub_i32 s0, 0, s15
; GFX90A-NEXT: v_rcp_f32_e32 v0, v0
; GFX90A-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX90A-NEXT: v_cvt_u32_f32_e32 v1, v0
-; GFX90A-NEXT: v_cvt_f32_f16_e32 v0, s0
-; GFX90A-NEXT: v_readfirstlane_b32 s2, v1
-; GFX90A-NEXT: s_mul_i32 s1, s1, s2
-; GFX90A-NEXT: s_mul_hi_u32 s1, s2, s1
-; GFX90A-NEXT: s_add_i32 s2, s2, s1
-; GFX90A-NEXT: s_mul_hi_u32 s1, s6, s2
-; GFX90A-NEXT: s_mul_i32 s2, s1, s7
-; GFX90A-NEXT: s_sub_i32 s2, s6, s2
-; GFX90A-NEXT: s_add_i32 s3, s1, 1
-; GFX90A-NEXT: s_sub_i32 s6, s2, s7
-; GFX90A-NEXT: s_cmp_ge_u32 s2, s7
+; GFX90A-NEXT: v_cvt_f32_f16_e32 v0, s18
+; GFX90A-NEXT: v_readfirstlane_b32 s1, v1
+; GFX90A-NEXT: s_mul_i32 s0, s0, s1
+; GFX90A-NEXT: s_mul_hi_u32 s0, s1, s0
+; GFX90A-NEXT: s_add_i32 s1, s1, s0
+; GFX90A-NEXT: s_mul_hi_u32 s0, s14, s1
+; GFX90A-NEXT: s_mul_i32 s1, s0, s15
+; GFX90A-NEXT: s_sub_i32 s1, s14, s1
+; GFX90A-NEXT: s_add_i32 s2, s0, 1
+; GFX90A-NEXT: s_sub_i32 s3, s1, s15
+; GFX90A-NEXT: s_cmp_ge_u32 s1, s15
+; GFX90A-NEXT: s_cselect_b32 s0, s2, s0
; GFX90A-NEXT: s_cselect_b32 s1, s3, s1
-; GFX90A-NEXT: s_cselect_b32 s2, s6, s2
-; GFX90A-NEXT: s_add_i32 s3, s1, 1
-; GFX90A-NEXT: s_cmp_ge_u32 s2, s7
-; GFX90A-NEXT: s_cselect_b32 s8, s3, s1
-; GFX90A-NEXT: s_lshr_b32 s2, s0, 16
+; GFX90A-NEXT: s_add_i32 s2, s0, 1
+; GFX90A-NEXT: s_cmp_ge_u32 s1, s15
+; GFX90A-NEXT: s_cselect_b32 s6, s2, s0
+; GFX90A-NEXT: s_lshr_b32 s2, s18, 16
; GFX90A-NEXT: v_cvt_f32_f16_e32 v1, s2
-; GFX90A-NEXT: s_lshl_b64 s[6:7], s[4:5], 5
-; GFX90A-NEXT: s_lshl_b64 s[14:15], s[10:11], 5
+; GFX90A-NEXT: s_lshl_b64 s[8:9], s[12:13], 5
+; GFX90A-NEXT: s_lshl_b64 s[10:11], s[16:17], 5
; GFX90A-NEXT: s_and_b64 s[0:1], exec, s[0:1]
-; GFX90A-NEXT: s_lshl_b64 s[16:17], s[8:9], 5
+; GFX90A-NEXT: s_lshl_b64 s[14:15], s[6:7], 5
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: v_readfirstlane_b32 s2, v18
; GFX90A-NEXT: s_and_b32 s2, 0xffff, s2
-; GFX90A-NEXT: s_mul_i32 s3, s5, s2
-; GFX90A-NEXT: s_mul_hi_u32 s5, s4, s2
-; GFX90A-NEXT: s_mul_i32 s2, s4, s2
+; GFX90A-NEXT: s_mul_i32 s3, s13, s2
+; GFX90A-NEXT: s_mul_hi_u32 s5, s12, s2
+; GFX90A-NEXT: s_mul_i32 s2, s12, s2
; GFX90A-NEXT: s_add_i32 s3, s5, s3
-; GFX90A-NEXT: s_lshl_b64 s[4:5], s[2:3], 5
+; GFX90A-NEXT: s_lshl_b64 s[12:13], s[2:3], 5
; GFX90A-NEXT: s_branch .LBB3_2
-; GFX90A-NEXT: .LBB3_1: ; %Flow20
+; GFX90A-NEXT: .LBB3_1: ; %Flow23
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; GFX90A-NEXT: s_cbranch_vccz .LBB3_12
@@ -731,36 +727,36 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: ; %bb.3: ; %bb14
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX90A-NEXT: global_load_dwordx2 v[4:5], v[2:3], off
-; GFX90A-NEXT: s_cmp_lt_i32 s11, 0
-; GFX90A-NEXT: s_mov_b32 s13, s12
+; GFX90A-NEXT: s_cmp_lt_i32 s17, 0
+; GFX90A-NEXT: s_mov_b32 s5, s4
; GFX90A-NEXT: s_cselect_b64 s[18:19], -1, 0
-; GFX90A-NEXT: s_cmp_gt_i32 s11, -1
-; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[12:13], s[12:13] op_sel:[0,1]
+; GFX90A-NEXT: s_cmp_gt_i32 s17, -1
+; GFX90A-NEXT: v_pk_mov_b32 v[6:7], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[4:5], s[4:5] op_sel:[0,1]
+; GFX90A-NEXT: v_pk_mov_b32 v[10:11], s[4:5], s[4:5] op_sel:[0,1]
; GFX90A-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GFX90A-NEXT: v_pk_mov_b32 v[8:9], s[12:13], s[12:13] op_sel:[0,1]
-; GFX90A-NEXT: v_pk_mov_b32 v[10:11], s[12:13], s[12:13] op_sel:[0,1]
; GFX90A-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[2:3]
-; GFX90A-NEXT: s_mov_b64 s[20:21], s[14:15]
+; GFX90A-NEXT: s_mov_b64 s[20:21], s[10:11]
; GFX90A-NEXT: v_cmp_ne_u32_e64 s[2:3], 1, v12
; GFX90A-NEXT: v_pk_mov_b32 v[12:13], v[6:7], v[6:7] op_sel:[0,1]
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s9, v4
-; GFX90A-NEXT: v_readfirstlane_b32 s13, v5
-; GFX90A-NEXT: s_add_u32 s9, s9, 1
-; GFX90A-NEXT: s_addc_u32 s13, s13, 0
-; GFX90A-NEXT: s_mul_hi_u32 s22, s6, s9
-; GFX90A-NEXT: s_mul_i32 s13, s6, s13
-; GFX90A-NEXT: s_mul_i32 s23, s7, s9
-; GFX90A-NEXT: s_add_i32 s13, s22, s13
-; GFX90A-NEXT: s_mul_i32 s9, s6, s9
-; GFX90A-NEXT: s_add_i32 s13, s13, s23
+; GFX90A-NEXT: v_readfirstlane_b32 s5, v4
+; GFX90A-NEXT: v_readfirstlane_b32 s7, v5
+; GFX90A-NEXT: s_add_u32 s5, s5, 1
+; GFX90A-NEXT: s_addc_u32 s7, s7, 0
+; GFX90A-NEXT: s_mul_hi_u32 s22, s8, s5
+; GFX90A-NEXT: s_mul_i32 s7, s8, s7
+; GFX90A-NEXT: s_mul_i32 s23, s9, s5
+; GFX90A-NEXT: s_add_i32 s7, s22, s7
+; GFX90A-NEXT: s_mul_i32 s5, s8, s5
+; GFX90A-NEXT: s_add_i32 s7, s7, s23
; GFX90A-NEXT: s_branch .LBB3_5
; GFX90A-NEXT: .LBB3_4: ; %bb58
; GFX90A-NEXT: ; in Loop: Header=BB3_5 Depth=2
; GFX90A-NEXT: v_add_co_u32_sdwa v4, vcc, v4, v18 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_0
; GFX90A-NEXT: v_addc_co_u32_e32 v5, vcc, 0, v5, vcc
-; GFX90A-NEXT: s_add_u32 s20, s20, s4
-; GFX90A-NEXT: s_addc_u32 s21, s21, s5
+; GFX90A-NEXT: s_add_u32 s20, s20, s12
+; GFX90A-NEXT: s_addc_u32 s21, s21, s13
; GFX90A-NEXT: s_mov_b64 s[22:23], 0
; GFX90A-NEXT: v_cmp_lt_i32_e64 s[24:25], -1, v5
; GFX90A-NEXT: s_andn2_b64 vcc, exec, s[24:25]
@@ -768,8 +764,8 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: .LBB3_5: ; %bb16
; GFX90A-NEXT: ; Parent Loop BB3_2 Depth=1
; GFX90A-NEXT: ; => This Inner Loop Header: Depth=2
-; GFX90A-NEXT: s_add_u32 s22, s20, s9
-; GFX90A-NEXT: s_addc_u32 s23, s21, s13
+; GFX90A-NEXT: s_add_u32 s22, s20, s5
+; GFX90A-NEXT: s_addc_u32 s23, s21, s7
; GFX90A-NEXT: global_load_dword v21, v19, s[22:23] offset:16 glc
; GFX90A-NEXT: s_waitcnt vmcnt(0)
; GFX90A-NEXT: global_load_dword v20, v19, s[22:23] offset:20 glc
@@ -810,17 +806,17 @@ define amdgpu_kernel void @introduced_copy_to_sgpr(i64 %arg, i32 %arg1, i32 %arg
; GFX90A-NEXT: .LBB3_9: ; %loop.exit.guard
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX90A-NEXT: s_xor_b64 s[18:19], s[22:23], -1
-; GFX90A-NEXT: .LBB3_10: ; %Flow19
+; GFX90A-NEXT: .LBB3_10: ; %Flow22
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
; GFX90A-NEXT: s_mov_b64 s[2:3], -1
; GFX90A-NEXT: s_and_b64 vcc, exec, s[18:19]
; GFX90A-NEXT: s_cbranch_vccz .LBB3_1
; GFX90A-NEXT: ; %bb.11: ; %bb12
; GFX90A-NEXT: ; in Loop: Header=BB3_2 Depth=1
-; GFX90A-NEXT: s_add_u32 s10, s10, s8
-; GFX90A-NEXT: s_addc_u32 s11, s11, 0
-; GFX90A-NEXT: s_add_u32 s14, s14, s16
-; GFX90A-NEXT: s_addc_u32 s15, s15, s17
+; GFX90A-NEXT: s_add_u32 s16, s16, s6
+; GFX90A-NEXT: s_addc_u32 s17, s17, 0
+; GFX90A-NEXT: s_add_u32 s10, s10, s14
+; GFX90A-NEXT: s_addc_u32 s11, s11, s15
; GFX90A-NEXT: s_mov_b64 s[2:3], 0
; GFX90A-NEXT: s_branch .LBB3_1
; GFX90A-NEXT: .LBB3_12: ; %DummyReturnBlock
diff --git a/llvm/test/CodeGen/AMDGPU/amd.endpgm.ll b/llvm/test/CodeGen/AMDGPU/amd.endpgm.ll
index 306942b9997e9..11a24d127abe3 100644
--- a/llvm/test/CodeGen/AMDGPU/amd.endpgm.ll
+++ b/llvm/test/CodeGen/AMDGPU/amd.endpgm.ll
@@ -44,16 +44,14 @@ define void @test1() {
define amdgpu_kernel void @test2(ptr %p, i32 %x) {
; GFX9-SDAG-LABEL: test2:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_cmp_lt_i32 s0, 1
+; GFX9-SDAG-NEXT: s_cmp_lt_i32 s2, 1
; GFX9-SDAG-NEXT: s_cbranch_scc0 .LBB2_2
; GFX9-SDAG-NEXT: ; %bb.1: ; %else
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s2
; GFX9-SDAG-NEXT: flat_store_dword v[0:1], v2
; GFX9-SDAG-NEXT: s_endpgm
; GFX9-SDAG-NEXT: .LBB2_2: ; %then
@@ -78,16 +76,14 @@ define amdgpu_kernel void @test2(ptr %p, i32 %x) {
;
; GFX10-SDAG-LABEL: test2:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: s_cmp_lt_i32 s0, 1
+; GFX10-SDAG-NEXT: s_cmp_lt_i32 s2, 1
; GFX10-SDAG-NEXT: s_cbranch_scc0 .LBB2_2
; GFX10-SDAG-NEXT: ; %bb.1: ; %else
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, s2
; GFX10-SDAG-NEXT: flat_store_dword v[0:1], v2
; GFX10-SDAG-NEXT: s_endpgm
; GFX10-SDAG-NEXT: .LBB2_2: ; %then
@@ -112,15 +108,13 @@ define amdgpu_kernel void @test2(ptr %p, i32 %x) {
;
; GFX11-SDAG-LABEL: test2:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_cmp_lt_i32 s0, 1
+; GFX11-SDAG-NEXT: s_cmp_lt_i32 s2, 1
; GFX11-SDAG-NEXT: s_cbranch_scc0 .LBB2_2
; GFX11-SDAG-NEXT: ; %bb.1: ; %else
-; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, s2
; GFX11-SDAG-NEXT: flat_store_b32 v[0:1], v2
; GFX11-SDAG-NEXT: s_endpgm
; GFX11-SDAG-NEXT: .LBB2_2: ; %then
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.private-memory.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.private-memory.ll
index 57771a8df7d14..ed130b327369e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.private-memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.private-memory.ll
@@ -13,14 +13,14 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
; GCN-LABEL: {{^}}work_item_info:
; GCN-NOT: v0
-; GCN: s_load_dword [[IN:s[0-9]+]]
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
; GCN-NOT: v0
; GCN-ALLOCA: v_add_{{[iu]}}32_e32 [[RESULT:v[0-9]+]], vcc, v{{[0-9]+}}, v0
-; GCN-PROMOTE: s_cmp_eq_u32 [[IN]], 1
-; GCN-PROMOTE-NEXT: s_cselect_b32 [[SCC:s[0-9]+]], 1, 0
-; GCN-PROMOTE-NEXT: v_add_{{[iu]}}32_e32 [[RESULT:v[0-9]+]], vcc, [[SCC]], v0
+; GCN-PROMOTE: s_cmp_eq_u32 [[IN:s[0-9]+]], 1
+; GCN-PROMOTE: s_cselect_b32 [[SCC:s[0-9]+]], 1, 0
+; GCN-PROMOTE: v_add_{{[iu]}}32_e32 [[RESULT:v[0-9]+]], vcc, [[SCC]], v0
; GCN: buffer_store_dword [[RESULT]]
define amdgpu_kernel void @work_item_info(ptr addrspace(1) %out, i32 %in) {
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
index 2203ff8771b5f..227a05400336e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-codegenprepare-idiv.ll
@@ -40,33 +40,34 @@ define amdgpu_kernel void @udiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
; GFX6-LABEL: udiv_i32:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_sub_i32 s2, 0, s5
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3
+; GFX6-NEXT: s_sub_i32 s4, 0, s3
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s2, v0
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: v_mul_lo_u32 v1, s4, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s4, v0
-; GFX6-NEXT: v_readfirstlane_b32 s6, v0
-; GFX6-NEXT: s_mul_i32 s6, s6, s5
-; GFX6-NEXT: s_sub_i32 s4, s4, s6
-; GFX6-NEXT: s_sub_i32 s6, s4, s5
+; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_mul_i32 s0, s0, s3
+; GFX6-NEXT: s_sub_i32 s0, s2, s0
+; GFX6-NEXT: s_sub_i32 s1, s0, s3
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT: s_cmp_ge_u32 s4, s5
+; GFX6-NEXT: s_cmp_ge_u32 s0, s3
; GFX6-NEXT: s_cselect_b64 vcc, -1, 0
; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT: s_cselect_b32 s4, s6, s4
+; GFX6-NEXT: s_cselect_b32 s0, s1, s0
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT: s_cmp_ge_u32 s4, s5
+; GFX6-NEXT: s_cmp_ge_u32 s0, s3
; GFX6-NEXT: s_cselect_b64 vcc, -1, 0
; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: udiv_i32:
@@ -137,30 +138,31 @@ define amdgpu_kernel void @urem_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
; GFX6-LABEL: urem_i32:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_sub_i32 s2, 0, s5
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3
+; GFX6-NEXT: s_sub_i32 s4, 0, s3
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s2, v0
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: v_mul_lo_u32 v1, s4, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s4, v0
-; GFX6-NEXT: v_readfirstlane_b32 s6, v0
-; GFX6-NEXT: s_mul_i32 s6, s6, s5
-; GFX6-NEXT: s_sub_i32 s4, s4, s6
-; GFX6-NEXT: s_sub_i32 s6, s4, s5
-; GFX6-NEXT: s_cmp_ge_u32 s4, s5
-; GFX6-NEXT: s_cselect_b32 s4, s6, s4
-; GFX6-NEXT: s_sub_i32 s6, s4, s5
-; GFX6-NEXT: s_cmp_ge_u32 s4, s5
-; GFX6-NEXT: s_cselect_b32 s4, s6, s4
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_mul_i32 s0, s0, s3
+; GFX6-NEXT: s_sub_i32 s0, s2, s0
+; GFX6-NEXT: s_sub_i32 s1, s0, s3
+; GFX6-NEXT: s_cmp_ge_u32 s0, s3
+; GFX6-NEXT: s_cselect_b32 s0, s1, s0
+; GFX6-NEXT: s_sub_i32 s1, s0, s3
+; GFX6-NEXT: s_cmp_ge_u32 s0, s3
+; GFX6-NEXT: s_cselect_b32 s0, s1, s0
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: urem_i32:
@@ -240,39 +242,40 @@ define amdgpu_kernel void @sdiv_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
; GFX6-LABEL: sdiv_i32:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_abs_i32 s6, s5
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX6-NEXT: s_sub_i32 s2, 0, s6
-; GFX6-NEXT: s_abs_i32 s7, s4
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
+; GFX6-NEXT: s_abs_i32 s8, s3
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s8
+; GFX6-NEXT: s_sub_i32 s4, 0, s8
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_xor_b32 s1, s2, s3
; GFX6-NEXT: v_rcp_f32_e32 v0, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 31
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_ashr_i32 s1, s1, 31
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s2, v0
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: v_mul_lo_u32 v1, s4, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_abs_i32 s0, s2
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s7, v0
-; GFX6-NEXT: v_readfirstlane_b32 s5, v0
-; GFX6-NEXT: s_mul_i32 s5, s5, s6
-; GFX6-NEXT: s_sub_i32 s5, s7, s5
-; GFX6-NEXT: s_sub_i32 s7, s5, s6
+; GFX6-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX6-NEXT: v_readfirstlane_b32 s2, v0
+; GFX6-NEXT: s_mul_i32 s2, s2, s8
+; GFX6-NEXT: s_sub_i32 s0, s0, s2
+; GFX6-NEXT: s_sub_i32 s2, s0, s8
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT: s_cmp_ge_u32 s5, s6
+; GFX6-NEXT: s_cmp_ge_u32 s0, s8
; GFX6-NEXT: s_cselect_b64 vcc, -1, 0
; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT: s_cselect_b32 s5, s7, s5
+; GFX6-NEXT: s_cselect_b32 s0, s2, s0
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT: s_cmp_ge_u32 s5, s6
+; GFX6-NEXT: s_cmp_ge_u32 s0, s8
; GFX6-NEXT: s_cselect_b64 vcc, -1, 0
; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT: v_xor_b32_e32 v0, s4, v0
-; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s4, v0
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: v_xor_b32_e32 v0, s1, v0
+; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s1, v0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: sdiv_i32:
@@ -357,35 +360,36 @@ define amdgpu_kernel void @srem_i32(ptr addrspace(1) %out, i32 %x, i32 %y) {
; GFX6-LABEL: srem_i32:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_abs_i32 s5, s5
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_sub_i32 s2, 0, s5
-; GFX6-NEXT: s_abs_i32 s6, s4
-; GFX6-NEXT: s_ashr_i32 s4, s4, 31
+; GFX6-NEXT: s_abs_i32 s3, s3
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3
+; GFX6-NEXT: s_sub_i32 s4, 0, s3
+; GFX6-NEXT: s_abs_i32 s8, s2
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_rcp_f32_e32 v0, v0
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s2, v0
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: v_mul_lo_u32 v1, s4, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_ashr_i32 s0, s2, 31
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s6, v0
-; GFX6-NEXT: v_readfirstlane_b32 s7, v0
-; GFX6-NEXT: s_mul_i32 s7, s7, s5
-; GFX6-NEXT: s_sub_i32 s6, s6, s7
-; GFX6-NEXT: s_sub_i32 s7, s6, s5
-; GFX6-NEXT: s_cmp_ge_u32 s6, s5
-; GFX6-NEXT: s_cselect_b32 s6, s7, s6
-; GFX6-NEXT: s_sub_i32 s7, s6, s5
-; GFX6-NEXT: s_cmp_ge_u32 s6, s5
-; GFX6-NEXT: s_cselect_b32 s5, s7, s6
-; GFX6-NEXT: s_xor_b32 s5, s5, s4
-; GFX6-NEXT: s_sub_i32 s4, s5, s4
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: v_mul_hi_u32 v0, s8, v0
+; GFX6-NEXT: v_readfirstlane_b32 s1, v0
+; GFX6-NEXT: s_mul_i32 s1, s1, s3
+; GFX6-NEXT: s_sub_i32 s1, s8, s1
+; GFX6-NEXT: s_sub_i32 s2, s1, s3
+; GFX6-NEXT: s_cmp_ge_u32 s1, s3
+; GFX6-NEXT: s_cselect_b32 s1, s2, s1
+; GFX6-NEXT: s_sub_i32 s2, s1, s3
+; GFX6-NEXT: s_cmp_ge_u32 s1, s3
+; GFX6-NEXT: s_cselect_b32 s1, s2, s1
+; GFX6-NEXT: s_xor_b32 s1, s1, s0
+; GFX6-NEXT: s_sub_i32 s0, s1, s0
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: srem_i32:
@@ -448,36 +452,33 @@ define amdgpu_kernel void @udiv_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
;
; GFX6-LABEL: udiv_i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_lshr_b32 s1, s0, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s1
-; GFX6-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s0
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_lshr_b32 s3, s2, 16
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3
+; GFX6-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s2
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: udiv_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v3, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s1, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s1
-; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
+; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
; GFX9-NEXT: v_rcp_f32_e32 v2, v0
; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
@@ -485,7 +486,6 @@ define amdgpu_kernel void @udiv_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v4, vcc
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v3, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = udiv i16 %x, %y
@@ -519,37 +519,36 @@ define amdgpu_kernel void @urem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
;
; GFX6-LABEL: urem_i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_lshr_b32 s2, s6, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX6-NEXT: s_and_b32 s0, s6, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s0
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_lshr_b32 s3, s2, 16
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3
+; GFX6-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
+; GFX6-NEXT: s_mov_b32 s4, s0
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s3
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT: buffer_store_short v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: urem_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s3, s2, 16
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX9-NEXT: s_and_b32 s0, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s0
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s4
; GFX9-NEXT: v_rcp_f32_e32 v2, v0
; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
@@ -560,7 +559,6 @@ define amdgpu_kernel void @urem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc
; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = urem i16 %x, %y
@@ -596,34 +594,34 @@ define amdgpu_kernel void @sdiv_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
;
; GFX6-LABEL: sdiv_i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_ashr_i32 s4, s6, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_ashr_i32 s0, s2, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_sext_i32_i16 s1, s2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s1
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: s_xor_b32 s0, s1, s0
+; GFX6-NEXT: s_ashr_i32 s0, s0, 30
+; GFX6-NEXT: s_or_b32 s2, s0, 1
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
-; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX6-NEXT: s_cselect_b32 s0, s2, 0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GFX6-NEXT: buffer_store_short v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: sdiv_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_ashr_i32 s3, s2, 16
@@ -679,56 +677,56 @@ define amdgpu_kernel void @srem_i16(ptr addrspace(1) %out, i16 %x, i16 %y) {
;
; GFX6-LABEL: srem_i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_ashr_i32 s7, s6, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s7
-; GFX6-NEXT: s_sext_i32_i16 s2, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX6-NEXT: s_xor_b32 s2, s2, s7
+; GFX6-NEXT: s_ashr_i32 s3, s2, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX6-NEXT: s_sext_i32_i16 s4, s2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX6-NEXT: s_xor_b32 s4, s4, s3
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s2, s2, 30
-; GFX6-NEXT: s_or_b32 s4, s2, 1
+; GFX6-NEXT: s_ashr_i32 s4, s4, 30
+; GFX6-NEXT: s_or_b32 s8, s4, 1
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX6-NEXT: s_cselect_b32 s2, s4, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s7
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
-; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX6-NEXT: s_cselect_b32 s4, s8, 0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s3
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT: buffer_store_short v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: srem_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_ashr_i32 s7, s6, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s7
-; GFX9-NEXT: s_sext_i32_i16 s2, s6
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
-; GFX9-NEXT: s_xor_b32 s2, s2, s7
+; GFX9-NEXT: s_ashr_i32 s3, s2, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX9-NEXT: s_sext_i32_i16 s4, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX9-NEXT: s_xor_b32 s4, s4, s3
; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_or_b32 s4, s2, 1
+; GFX9-NEXT: s_ashr_i32 s4, s4, 30
+; GFX9-NEXT: s_or_b32 s6, s4, 1
; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v2
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s7
+; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s4, s6, 0
+; GFX9-NEXT: v_add_u32_e32 v0, s4, v2
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
; GFX9-NEXT: global_store_short v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = srem i16 %x, %y
@@ -760,27 +758,27 @@ define amdgpu_kernel void @udiv_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
;
; GFX6-LABEL: udiv_i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cvt_f32_ubyte1_e32 v0, s6
+; GFX6-NEXT: v_cvt_f32_ubyte1_e32 v0, s2
; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
; GFX6-NEXT: v_mad_f32 v1, -v1, v0, v2
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
-; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: udiv_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v0, s2
@@ -825,30 +823,30 @@ define amdgpu_kernel void @urem_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
;
; GFX6-LABEL: urem_i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cvt_f32_ubyte1_e32 v0, s6
+; GFX6-NEXT: v_cvt_f32_ubyte1_e32 v0, s2
; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s6
-; GFX6-NEXT: s_lshr_b32 s2, s6, 8
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s2
+; GFX6-NEXT: s_lshr_b32 s3, s2, 8
+; GFX6-NEXT: s_mov_b32 s4, s0
; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
; GFX6-NEXT: v_mad_f32 v1, -v1, v0, v2
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
-; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s3
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: urem_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_cvt_f32_ubyte1_e32 v0, s2
; GFX9-NEXT: v_rcp_f32_e32 v1, v0
@@ -898,34 +896,34 @@ define amdgpu_kernel void @sdiv_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
;
; GFX6-LABEL: sdiv_i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s4, s6, 0x80008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i8 s5, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_bfe_i32 s0, s2, 0x80008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_sext_i32_i8 s1, s2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s1
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: s_xor_b32 s0, s1, s0
+; GFX6-NEXT: s_ashr_i32 s0, s0, 30
+; GFX6-NEXT: s_or_b32 s2, s0, 1
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
-; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX6-NEXT: s_cselect_b32 s0, s2, 0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: sdiv_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_i32 s3, s2, 0x80008
@@ -981,58 +979,58 @@ define amdgpu_kernel void @srem_i8(ptr addrspace(1) %out, i8 %x, i8 %y) {
;
; GFX6-LABEL: srem_i8:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s2, s6, 0x80008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX6-NEXT: s_sext_i32_i8 s3, s6
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s3
-; GFX6-NEXT: s_xor_b32 s2, s3, s2
+; GFX6-NEXT: s_bfe_i32 s3, s2, 0x80008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX6-NEXT: s_sext_i32_i8 s4, s2
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX6-NEXT: s_xor_b32 s3, s4, s3
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s2, s2, 30
-; GFX6-NEXT: s_lshr_b32 s4, s6, 8
-; GFX6-NEXT: s_or_b32 s5, s2, 1
+; GFX6-NEXT: s_ashr_i32 s3, s3, 30
+; GFX6-NEXT: s_lshr_b32 s8, s2, 8
+; GFX6-NEXT: s_or_b32 s3, s3, 1
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX6-NEXT: s_cselect_b32 s2, s5, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
-; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX6-NEXT: s_cselect_b32 s3, s3, 0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s3, v2
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s8
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: srem_i8:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s2, s6, 0x80008
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: s_sext_i32_i8 s3, s6
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s3
-; GFX9-NEXT: s_xor_b32 s2, s3, s2
+; GFX9-NEXT: s_bfe_i32 s3, s2, 0x80008
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX9-NEXT: s_sext_i32_i8 s4, s2
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX9-NEXT: s_xor_b32 s3, s4, s3
; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
-; GFX9-NEXT: s_lshr_b32 s4, s6, 8
-; GFX9-NEXT: s_or_b32 s5, s2, 1
+; GFX9-NEXT: s_ashr_i32 s3, s3, 30
+; GFX9-NEXT: s_lshr_b32 s6, s2, 8
+; GFX9-NEXT: s_or_b32 s3, s3, 1
; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX9-NEXT: s_cselect_b32 s2, s5, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v2
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s4
+; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s3, s3, 0
+; GFX9-NEXT: v_add_u32_e32 v0, s3, v2
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_sub_u32_e32 v0, s6, v0
+; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = srem i8 %x, %y
@@ -2543,44 +2541,45 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
;
; GFX6-LABEL: udiv_v4i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX6-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
-; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
+; GFX6-NEXT: s_and_b32 s0, s8, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s0
+; GFX6-NEXT: s_mov_b32 s0, s4
+; GFX6-NEXT: s_and_b32 s1, s6, 0xffff
; GFX6-NEXT: s_lshr_b32 s4, s8, 16
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX6-NEXT: s_lshr_b32 s4, s6, 16
; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
+; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
+; GFX6-NEXT: v_rcp_f32_e32 v5, v3
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
+; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
+; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: s_and_b32 s4, s11, 0xffff
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
-; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v4
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
; GFX6-NEXT: s_and_b32 s4, s9, 0xffff
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
+; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; GFX6-NEXT: v_mad_f32 v2, -v1, v3, v4
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX6-NEXT: s_and_b32 s4, s7, 0xffff
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4
; GFX6-NEXT: v_rcp_f32_e32 v6, v4
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
+; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v1, vcc
; GFX6-NEXT: v_mul_f32_e32 v1, v5, v6
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: s_lshr_b32 s4, s11, 16
+; GFX6-NEXT: s_lshr_b32 s4, s9, 16
; GFX6-NEXT: v_mad_f32 v3, -v1, v4, v5
; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4
-; GFX6-NEXT: s_lshr_b32 s4, s9, 16
+; GFX6-NEXT: s_lshr_b32 s4, s7, 16
; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s4
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX6-NEXT: v_rcp_f32_e32 v7, v5
@@ -2596,6 +2595,7 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX6-NEXT: v_addc_u32_e32 v3, vcc, 0, v4, vcc
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: s_mov_b32 s1, s5
; GFX6-NEXT: v_or_b32_e32 v1, v1, v3
; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
@@ -2603,42 +2603,43 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
;
; GFX9-LABEL: udiv_v4i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_and_b32 s6, s0, 0xffff
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s6
+; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s6, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
+; GFX9-NEXT: s_lshr_b32 s6, s6, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s4
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s6
; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
-; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s2
; GFX9-NEXT: v_rcp_f32_e32 v5, v1
+; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: s_and_b32 s0, s3, 0xffff
+; GFX9-NEXT: s_and_b32 s2, s7, 0xffff
; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4
; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s2
; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: s_and_b32 s0, s1, 0xffff
+; GFX9-NEXT: s_and_b32 s2, s3, 0xffff
; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
; GFX9-NEXT: v_trunc_f32_e32 v2, v5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s2
; GFX9-NEXT: v_rcp_f32_e32 v7, v4
; GFX9-NEXT: v_mad_f32 v3, -v2, v1, v3
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v1
-; GFX9-NEXT: s_lshr_b32 s0, s3, 16
+; GFX9-NEXT: s_lshr_b32 s2, s7, 16
; GFX9-NEXT: v_mul_f32_e32 v1, v5, v7
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_mad_f32 v3, -v1, v4, v5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s2
; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT: s_lshr_b32 s0, s1, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s0
+; GFX9-NEXT: s_lshr_b32 s2, s3, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s2
; GFX9-NEXT: v_rcp_f32_e32 v8, v5
; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v1
@@ -2646,7 +2647,6 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_mul_f32_e32 v3, v7, v8
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v3
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; GFX9-NEXT: v_mad_f32 v3, -v3, v5, v7
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v5
@@ -2655,8 +2655,7 @@ define amdgpu_kernel void @udiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
; GFX9-NEXT: v_lshl_or_b32 v0, v2, 16, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7]
+; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
%r = udiv <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -2758,49 +2757,51 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
;
; GFX6-LABEL: urem_v4i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX6-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
-; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
+; GFX6-NEXT: s_and_b32 s0, s8, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s0
+; GFX6-NEXT: s_mov_b32 s0, s4
+; GFX6-NEXT: s_and_b32 s1, s6, 0xffff
; GFX6-NEXT: s_lshr_b32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX6-NEXT: s_mov_b32 s1, s5
+; GFX6-NEXT: s_lshr_b32 s5, s6, 16
+; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s5
+; GFX6-NEXT: v_rcp_f32_e32 v5, v3
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
-; GFX6-NEXT: v_mad_f32 v1, -v1, v2, v4
-; GFX6-NEXT: s_and_b32 s6, s11, 0xffff
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v2
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s6
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s5
-; GFX6-NEXT: s_and_b32 s5, s9, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s5
+; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v1
+; GFX6-NEXT: v_mad_f32 v1, -v1, v3, v4
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s8
+; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v3
+; GFX6-NEXT: s_and_b32 s8, s9, 0xffff
+; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s8
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s4
+; GFX6-NEXT: s_and_b32 s4, s7, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
; GFX6-NEXT: v_rcp_f32_e32 v4, v2
-; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s4, v1
-; GFX6-NEXT: s_lshr_b32 s4, s11, 16
+; GFX6-NEXT: s_lshr_b32 s4, s9, 16
+; GFX6-NEXT: v_sub_i32_e32 v5, vcc, s5, v1
; GFX6-NEXT: v_mul_f32_e32 v1, v3, v4
; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
-; GFX6-NEXT: s_lshr_b32 s5, s9, 16
+; GFX6-NEXT: s_lshr_b32 s5, s7, 16
; GFX6-NEXT: v_cvt_f32_u32_e32 v6, s5
-; GFX6-NEXT: v_rcp_f32_e32 v7, v4
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT: v_rcp_f32_e32 v7, v4
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v3
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
@@ -2811,10 +2812,10 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX6-NEXT: v_mad_f32 v2, -v2, v4, v6
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s11
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s9
; GFX6-NEXT: v_mul_lo_u32 v2, v2, s4
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s9, v1
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s7, v1
; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s5, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -2826,30 +2827,30 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
;
; GFX9-LABEL: urem_v4i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s9, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s9
-; GFX9-NEXT: s_and_b32 s8, s0, 0xffff
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s8
+; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s6, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
+; GFX9-NEXT: s_lshr_b32 s8, s6, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s8
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s4
; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s2
; GFX9-NEXT: v_rcp_f32_e32 v5, v1
; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: s_and_b32 s4, s3, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s7, 0xffff
; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4
; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s5
; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
; GFX9-NEXT: v_trunc_f32_e32 v2, v5
-; GFX9-NEXT: s_and_b32 s5, s1, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s3, 0xffff
; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
; GFX9-NEXT: v_mad_f32 v3, -v2, v1, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
@@ -2858,35 +2859,35 @@ define amdgpu_kernel void @urem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v1
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
; GFX9-NEXT: v_mul_f32_e32 v2, v5, v7
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: s_lshr_b32 s2, s3, 16
+; GFX9-NEXT: s_lshr_b32 s6, s7, 16
; GFX9-NEXT: v_mad_f32 v3, -v2, v4, v5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s2
-; GFX9-NEXT: s_lshr_b32 s1, s1, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s6
+; GFX9-NEXT: s_lshr_b32 s3, s3, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v7, s3
; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX9-NEXT: v_rcp_f32_e32 v8, v5
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v4
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s9
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s8
; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
; GFX9-NEXT: v_mul_f32_e32 v3, v7, v8
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v3
; GFX9-NEXT: v_mad_f32 v3, -v3, v5, v7
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v5
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s4
+; GFX9-NEXT: v_mul_lo_u32 v2, v2, s7
; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v4, vcc
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, s2
-; GFX9-NEXT: v_sub_u32_e32 v0, s8, v0
-; GFX9-NEXT: v_sub_u32_e32 v4, s0, v1
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, s6
+; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
+; GFX9-NEXT: v_sub_u32_e32 v4, s2, v1
; GFX9-NEXT: v_sub_u32_e32 v1, s5, v2
-; GFX9-NEXT: v_sub_u32_e32 v2, s1, v3
+; GFX9-NEXT: v_sub_u32_e32 v2, s3, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
-; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[6:7]
+; GFX9-NEXT: global_store_dwordx2 v6, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
%r = urem <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -2996,62 +2997,64 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
;
; GFX6-LABEL: sdiv_v4i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_sext_i32_i16 s4, s10
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
+; GFX6-NEXT: s_sext_i32_i16 s7, s10
+; GFX6-NEXT: s_sext_i32_i16 s6, s4
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s6
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s7
+; GFX6-NEXT: s_xor_b32 s6, s7, s6
+; GFX6-NEXT: s_ashr_i32 s6, s6, 30
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: s_mov_b32 s0, s8
+; GFX6-NEXT: s_or_b32 s8, s6, 1
+; GFX6-NEXT: s_mov_b32 s1, s9
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[6:7], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: s_ashr_i32 s5, s10, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX6-NEXT: s_cselect_b32 s6, s8, 0
+; GFX6-NEXT: s_ashr_i32 s4, s4, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, s6, v2
+; GFX6-NEXT: s_ashr_i32 s6, s10, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s6
; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
+; GFX6-NEXT: s_xor_b32 s4, s6, s4
; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: s_or_b32 s4, s4, 1
; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
; GFX6-NEXT: v_trunc_f32_e32 v3, v3
; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[6:7], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: s_sext_i32_i16 s5, s11
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
+; GFX6-NEXT: s_sext_i32_i16 s6, s5
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s6
+; GFX6-NEXT: s_cselect_b32 s4, s4, 0
; GFX6-NEXT: v_add_i32_e32 v3, vcc, s4, v3
-; GFX6-NEXT: s_sext_i32_i16 s4, s9
+; GFX6-NEXT: s_sext_i32_i16 s4, s11
; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
; GFX6-NEXT: v_rcp_f32_e32 v4, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
+; GFX6-NEXT: s_xor_b32 s4, s4, s6
; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: s_or_b32 s4, s4, 1
; GFX6-NEXT: v_mul_f32_e32 v4, v1, v4
; GFX6-NEXT: v_trunc_f32_e32 v4, v4
; GFX6-NEXT: v_mad_f32 v1, -v4, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[6:7], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: s_ashr_i32 s5, s11, 16
+; GFX6-NEXT: s_cselect_b32 s4, s4, 0
+; GFX6-NEXT: s_ashr_i32 s5, s5, 16
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v4
-; GFX6-NEXT: s_ashr_i32 s4, s9, 16
+; GFX6-NEXT: s_ashr_i32 s4, s11, 16
; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s4
; GFX6-NEXT: v_rcp_f32_e32 v5, v0
; GFX6-NEXT: s_xor_b32 s4, s4, s5
@@ -3076,13 +3079,13 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
;
; GFX9-LABEL: sdiv_v4i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s4, s2
+; GFX9-NEXT: s_sext_i32_i16 s4, s6
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX9-NEXT: s_sext_i32_i16 s5, s0
+; GFX9-NEXT: s_sext_i32_i16 s5, s2
; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s5
; GFX9-NEXT: s_xor_b32 s4, s5, s4
; GFX9-NEXT: v_rcp_f32_e32 v3, v0
@@ -3094,61 +3097,61 @@ define amdgpu_kernel void @sdiv_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX9-NEXT: s_cselect_b32 s4, s8, 0
+; GFX9-NEXT: s_ashr_i32 s5, s6, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX9-NEXT: s_ashr_i32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_sext_i32_i16 s2, s3
+; GFX9-NEXT: s_xor_b32 s2, s2, s5
+; GFX9-NEXT: s_ashr_i32 s2, s2, 30
+; GFX9-NEXT: v_add_u32_e32 v3, s4, v3
; GFX9-NEXT: v_mul_f32_e32 v4, v1, v4
; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_mad_f32 v1, -v4, v0, v1
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_add_u32_e32 v3, s4, v3
-; GFX9-NEXT: s_or_b32 s0, s0, 1
+; GFX9-NEXT: s_or_b32 s2, s2, 1
; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: v_add_u32_e32 v4, s0, v4
-; GFX9-NEXT: s_sext_i32_i16 s0, s1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
+; GFX9-NEXT: s_sext_i32_i16 s4, s7
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s4
+; GFX9-NEXT: s_cselect_b32 s2, s2, 0
+; GFX9-NEXT: v_add_u32_e32 v4, s2, v4
+; GFX9-NEXT: s_sext_i32_i16 s2, s3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s2
; GFX9-NEXT: v_rcp_f32_e32 v5, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s0, s0, 1
+; GFX9-NEXT: s_xor_b32 s2, s2, s4
+; GFX9-NEXT: s_ashr_i32 s2, s2, 30
+; GFX9-NEXT: s_or_b32 s2, s2, 1
; GFX9-NEXT: v_mul_f32_e32 v5, v1, v5
; GFX9-NEXT: v_trunc_f32_e32 v5, v5
; GFX9-NEXT: v_mad_f32 v1, -v5, v0, v1
; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
+; GFX9-NEXT: s_cselect_b32 s2, s2, 0
+; GFX9-NEXT: s_ashr_i32 s4, s7, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s4
+; GFX9-NEXT: v_add_u32_e32 v1, s2, v5
; GFX9-NEXT: s_ashr_i32 s2, s3, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: v_add_u32_e32 v1, s0, v5
-; GFX9-NEXT: s_ashr_i32 s0, s1, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s2
; GFX9-NEXT: v_rcp_f32_e32 v6, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s2, s0, 1
+; GFX9-NEXT: s_xor_b32 s2, s2, s4
+; GFX9-NEXT: s_ashr_i32 s2, s2, 30
+; GFX9-NEXT: s_or_b32 s4, s2, 1
; GFX9-NEXT: v_mul_f32_e32 v6, v5, v6
; GFX9-NEXT: v_trunc_f32_e32 v6, v6
; GFX9-NEXT: v_mad_f32 v5, -v6, v0, v5
; GFX9-NEXT: v_cvt_i32_f32_e32 v6, v6
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v5|, |v0|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s0, v6
+; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v5|, |v0|
+; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cselect_b32 s2, s4, 0
+; GFX9-NEXT: v_add_u32_e32 v0, s2, v6
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_lshl_or_b32 v1, v0, 16, v1
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v3
; GFX9-NEXT: v_lshl_or_b32 v0, v4, 16, v0
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
%r = sdiv <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -3266,53 +3269,54 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
;
; GFX6-LABEL: srem_v4i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX6-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_sext_i32_i16 s4, s10
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
+; GFX6-NEXT: s_sext_i32_i16 s0, s8
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX6-NEXT: s_sext_i32_i16 s1, s6
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s1
+; GFX6-NEXT: s_xor_b32 s0, s1, s0
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: s_ashr_i32 s0, s0, 30
+; GFX6-NEXT: s_or_b32 s10, s0, 1
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s10, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: s_ashr_i32 s5, s8, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s5
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX6-NEXT: s_cselect_b32 s0, s10, 0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s8
+; GFX6-NEXT: s_ashr_i32 s8, s8, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s8
+; GFX6-NEXT: s_mov_b32 s0, s4
+; GFX6-NEXT: s_ashr_i32 s4, s6, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
; GFX6-NEXT: v_rcp_f32_e32 v3, v1
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
+; GFX6-NEXT: s_xor_b32 s4, s4, s8
; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_lshr_b32 s6, s8, 16
+; GFX6-NEXT: s_mov_b32 s1, s5
; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3
; GFX6-NEXT: v_trunc_f32_e32 v3, v3
; GFX6-NEXT: v_mad_f32 v2, -v3, v1, v2
; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
-; GFX6-NEXT: s_lshr_b32 s7, s10, 16
-; GFX6-NEXT: s_or_b32 s8, s4, 1
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
+; GFX6-NEXT: s_lshr_b32 s10, s6, 16
+; GFX6-NEXT: s_or_b32 s6, s4, 1
; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v1|
; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s8, 0
+; GFX6-NEXT: s_cselect_b32 s4, s6, 0
; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v3
-; GFX6-NEXT: s_sext_i32_i16 s4, s11
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s7
+; GFX6-NEXT: s_sext_i32_i16 s4, s9
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s8
; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s9
+; GFX6-NEXT: s_sext_i32_i16 s5, s7
; GFX6-NEXT: s_xor_b32 s4, s5, s4
-; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s6, v1
+; GFX6-NEXT: v_sub_i32_e32 v3, vcc, s10, v1
; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
; GFX6-NEXT: v_rcp_f32_e32 v4, v2
; GFX6-NEXT: s_ashr_i32 s4, s4, 30
@@ -3321,35 +3325,34 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
; GFX6-NEXT: v_mul_f32_e32 v4, v1, v4
; GFX6-NEXT: v_trunc_f32_e32 v4, v4
; GFX6-NEXT: v_mad_f32 v1, -v4, v2, v1
-; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v2|
; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
; GFX6-NEXT: s_cselect_b32 s4, s6, 0
+; GFX6-NEXT: s_ashr_i32 s6, s9, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s6
; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v4
-; GFX6-NEXT: s_ashr_i32 s4, s11, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: s_ashr_i32 s5, s9, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
+; GFX6-NEXT: s_ashr_i32 s4, s7, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v4, s4
; GFX6-NEXT: v_rcp_f32_e32 v5, v2
+; GFX6-NEXT: s_xor_b32 s4, s4, s6
; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_lshr_b32 s6, s9, 16
-; GFX6-NEXT: s_lshr_b32 s7, s11, 16
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s9
; GFX6-NEXT: v_mul_f32_e32 v5, v4, v5
; GFX6-NEXT: v_trunc_f32_e32 v5, v5
; GFX6-NEXT: v_mad_f32 v4, -v5, v2, v4
; GFX6-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX6-NEXT: s_or_b32 s8, s4, 1
+; GFX6-NEXT: s_lshr_b32 s8, s7, 16
+; GFX6-NEXT: s_or_b32 s9, s4, 1
; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, |v2|
; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s8, 0
+; GFX6-NEXT: s_cselect_b32 s4, s9, 0
; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v5
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s11
-; GFX6-NEXT: v_mul_lo_u32 v2, v2, s7
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s9, v1
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s6, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_mul_lo_u32 v2, v2, s6
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s7, v1
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s8, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
; GFX6-NEXT: v_or_b32_e32 v0, v0, v2
@@ -3358,87 +3361,87 @@ define amdgpu_kernel void @srem_v4i16(ptr addrspace(1) %out, <4 x i16> %x, <4 x
;
; GFX9-LABEL: srem_v4i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_sext_i32_i16 s4, s6
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s4
; GFX9-NEXT: s_sext_i32_i16 s8, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s8
-; GFX9-NEXT: s_sext_i32_i16 s9, s0
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s9
-; GFX9-NEXT: s_xor_b32 s4, s9, s8
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s8
+; GFX9-NEXT: s_xor_b32 s4, s8, s4
; GFX9-NEXT: v_rcp_f32_e32 v3, v0
; GFX9-NEXT: s_ashr_i32 s4, s4, 30
-; GFX9-NEXT: s_or_b32 s10, s4, 1
+; GFX9-NEXT: s_or_b32 s9, s4, 1
; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_mad_f32 v1, -v3, v0, v1
; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s4, s10, 0
-; GFX9-NEXT: s_ashr_i32 s10, s0, 16
-; GFX9-NEXT: s_ashr_i32 s0, s2, 16
+; GFX9-NEXT: s_cselect_b32 s4, s9, 0
+; GFX9-NEXT: s_ashr_i32 s9, s2, 16
+; GFX9-NEXT: s_ashr_i32 s2, s6, 16
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9-NEXT: s_xor_b32 s2, s10, s0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
; GFX9-NEXT: v_add_u32_e32 v1, s4, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s10
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s9
; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_or_b32 s2, s2, 1
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s8
-; GFX9-NEXT: s_sext_i32_i16 s8, s1
+; GFX9-NEXT: s_xor_b32 s4, s9, s2
+; GFX9-NEXT: s_ashr_i32 s4, s4, 30
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s6
; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4
; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_mad_f32 v3, -v4, v0, v3
; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
+; GFX9-NEXT: s_or_b32 s6, s4, 1
; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s2, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v4
-; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s8
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s0
-; GFX9-NEXT: s_xor_b32 s0, s8, s2
+; GFX9-NEXT: s_cselect_b32 s4, s6, 0
+; GFX9-NEXT: v_add_u32_e32 v0, s4, v4
+; GFX9-NEXT: s_sext_i32_i16 s4, s7
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s4
+; GFX9-NEXT: s_sext_i32_i16 s6, s3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s6
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX9-NEXT: v_rcp_f32_e32 v5, v3
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s0, s0, 1
-; GFX9-NEXT: v_sub_u32_e32 v0, s10, v0
+; GFX9-NEXT: s_xor_b32 s2, s6, s4
+; GFX9-NEXT: s_ashr_i32 s2, s2, 30
+; GFX9-NEXT: s_or_b32 s2, s2, 1
; GFX9-NEXT: v_mul_f32_e32 v5, v4, v5
; GFX9-NEXT: v_trunc_f32_e32 v5, v5
; GFX9-NEXT: v_mad_f32 v4, -v5, v3, v4
; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, |v3|
-; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: s_ashr_i32 s3, s3, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s3
-; GFX9-NEXT: v_add_u32_e32 v3, s0, v5
-; GFX9-NEXT: v_mul_lo_u32 v3, v3, s2
-; GFX9-NEXT: s_ashr_i32 s2, s1, 16
-; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s2
+; GFX9-NEXT: s_cselect_b32 s2, s2, 0
+; GFX9-NEXT: s_ashr_i32 s4, s7, 16
+; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s4
+; GFX9-NEXT: s_ashr_i32 s5, s3, 16
+; GFX9-NEXT: v_sub_u32_e32 v0, s9, v0
+; GFX9-NEXT: v_add_u32_e32 v3, s2, v5
+; GFX9-NEXT: v_cvt_f32_i32_e32 v5, s5
; GFX9-NEXT: v_rcp_f32_e32 v6, v4
-; GFX9-NEXT: s_xor_b32 s0, s2, s3
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s4, s0, 1
+; GFX9-NEXT: s_xor_b32 s2, s5, s4
+; GFX9-NEXT: s_ashr_i32 s2, s2, 30
+; GFX9-NEXT: v_mul_lo_u32 v3, v3, s7
; GFX9-NEXT: v_mul_f32_e32 v6, v5, v6
; GFX9-NEXT: v_trunc_f32_e32 v6, v6
; GFX9-NEXT: v_mad_f32 v5, -v6, v4, v5
; GFX9-NEXT: v_cvt_i32_f32_e32 v6, v6
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v5|, |v4|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v4, s0, v6
-; GFX9-NEXT: v_mul_lo_u32 v4, v4, s3
-; GFX9-NEXT: v_sub_u32_e32 v5, s9, v1
-; GFX9-NEXT: v_sub_u32_e32 v1, s8, v3
+; GFX9-NEXT: s_or_b32 s7, s2, 1
+; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v5|, |v4|
+; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cselect_b32 s2, s7, 0
+; GFX9-NEXT: v_add_u32_e32 v4, s2, v6
+; GFX9-NEXT: v_mul_lo_u32 v4, v4, s4
+; GFX9-NEXT: v_sub_u32_e32 v5, s8, v1
+; GFX9-NEXT: v_sub_u32_e32 v1, s6, v3
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NEXT: v_sub_u32_e32 v3, s2, v4
+; GFX9-NEXT: v_sub_u32_e32 v3, s5, v4
; GFX9-NEXT: v_lshl_or_b32 v1, v3, 16, v1
; GFX9-NEXT: v_and_b32_e32 v3, 0xffff, v5
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
%r = srem <4 x i16> %x, %y
store <4 x i16> %r, ptr addrspace(1) %out
@@ -3469,16 +3472,17 @@ define amdgpu_kernel void @udiv_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
;
; GFX6-LABEL: udiv_i3:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_u32 s2, s6, 0x30008
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_bfe_u32 s0, s2, 0x30008
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: s_and_b32 s4, s6, 7
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s4
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_and_b32 s0, s2, 7
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
@@ -3486,13 +3490,12 @@ define amdgpu_kernel void @udiv_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
-; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: udiv_i3:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_u32 s3, s2, 0x30008
@@ -3540,39 +3543,40 @@ define amdgpu_kernel void @urem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
;
; GFX6-LABEL: urem_i3:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_u32 s2, s6, 0x30008
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, s2
+; GFX6-NEXT: s_bfe_u32 s3, s2, 0x30008
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
; GFX6-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-NEXT: s_and_b32 s3, s6, 7
-; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s3
-; GFX6-NEXT: s_lshr_b32 s2, s6, 8
+; GFX6-NEXT: s_and_b32 s4, s2, 7
+; GFX6-NEXT: v_cvt_f32_ubyte0_e32 v2, s4
+; GFX6-NEXT: s_lshr_b32 s3, s2, 8
; GFX6-NEXT: v_mul_f32_e32 v1, v2, v1
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v1
; GFX6-NEXT: v_mad_f32 v1, -v1, v0, v2
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_mov_b32 s4, s0
; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s3
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
-; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: urem_i3:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_u32 s0, s2, 0x30008
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s0
+; GFX9-NEXT: s_bfe_u32 s3, s2, 0x30008
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v0, s3
; GFX9-NEXT: v_rcp_f32_e32 v1, v0
-; GFX9-NEXT: s_and_b32 s1, s2, 7
-; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s1
-; GFX9-NEXT: s_lshr_b32 s0, s2, 8
+; GFX9-NEXT: s_and_b32 s4, s2, 7
+; GFX9-NEXT: v_cvt_f32_ubyte0_e32 v2, s4
+; GFX9-NEXT: s_lshr_b32 s3, s2, 8
; GFX9-NEXT: v_mul_f32_e32 v1, v2, v1
; GFX9-NEXT: v_trunc_f32_e32 v1, v1
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v1
@@ -3580,11 +3584,9 @@ define amdgpu_kernel void @urem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v3, vcc
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s0
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
; GFX9-NEXT: v_and_b32_e32 v0, 7, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = urem i3 %x, %y
@@ -3620,35 +3622,35 @@ define amdgpu_kernel void @sdiv_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
;
; GFX6-LABEL: sdiv_i3:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s4, s6, 0x30008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_bfe_i32 s5, s6, 0x30000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_bfe_i32 s0, s2, 0x30008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_bfe_i32 s1, s2, 0x30000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s1
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: s_xor_b32 s0, s1, s0
+; GFX6-NEXT: s_ashr_i32 s0, s0, 30
+; GFX6-NEXT: s_or_b32 s2, s0, 1
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX6-NEXT: s_cselect_b32 s0, s2, 0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v2
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
-; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: sdiv_i3:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_i32 s3, s2, 0x30008
@@ -3705,61 +3707,60 @@ define amdgpu_kernel void @srem_i3(ptr addrspace(1) %out, i3 %x, i3 %y) {
;
; GFX6-LABEL: srem_i3:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_bfe_i32 s2, s6, 0x30008
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX6-NEXT: s_bfe_i32 s3, s6, 0x30000
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s3
-; GFX6-NEXT: s_xor_b32 s2, s3, s2
+; GFX6-NEXT: s_bfe_i32 s3, s2, 0x30008
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX6-NEXT: s_bfe_i32 s4, s2, 0x30000
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX6-NEXT: s_xor_b32 s3, s4, s3
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s2, s2, 30
-; GFX6-NEXT: s_lshr_b32 s4, s6, 8
-; GFX6-NEXT: s_or_b32 s5, s2, 1
+; GFX6-NEXT: s_ashr_i32 s3, s3, 30
+; GFX6-NEXT: s_lshr_b32 s8, s2, 8
+; GFX6-NEXT: s_or_b32 s3, s3, 1
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GFX6-NEXT: s_cselect_b32 s2, s5, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v2
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX6-NEXT: s_cselect_b32 s3, s3, 0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s3, v2
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s8
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0
; GFX6-NEXT: v_and_b32_e32 v0, 7, v0
-; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: srem_i3:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s0, s2, 0x30008
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9-NEXT: s_bfe_i32 s1, s2, 0x30000
-; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s1
-; GFX9-NEXT: s_xor_b32 s0, s1, s0
+; GFX9-NEXT: s_bfe_i32 s3, s2, 0x30008
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX9-NEXT: s_bfe_i32 s4, s2, 0x30000
+; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s4
+; GFX9-NEXT: s_xor_b32 s3, s4, s3
; GFX9-NEXT: v_rcp_f32_e32 v2, v0
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_lshr_b32 s3, s2, 8
-; GFX9-NEXT: s_or_b32 s6, s0, 1
+; GFX9-NEXT: s_ashr_i32 s3, s3, 30
+; GFX9-NEXT: s_lshr_b32 s6, s2, 8
+; GFX9-NEXT: s_or_b32 s3, s3, 1
; GFX9-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
; GFX9-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s6, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s0, v2
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s3
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
+; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX9-NEXT: s_cselect_b32 s3, s3, 0
+; GFX9-NEXT: v_add_u32_e32 v0, s3, v2
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s6
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: v_sub_u32_e32 v0, s2, v0
; GFX9-NEXT: v_and_b32_e32 v0, 7, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_byte v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = srem i3 %x, %y
@@ -3834,46 +3835,48 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
;
; GFX6-LABEL: udiv_v3i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX6-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
-; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
+; GFX6-NEXT: s_and_b32 s0, s8, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s0
+; GFX6-NEXT: s_mov_b32 s0, s4
+; GFX6-NEXT: s_and_b32 s1, s6, 0xffff
; GFX6-NEXT: s_lshr_b32 s4, s8, 16
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX6-NEXT: s_lshr_b32 s4, s6, 16
; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
+; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
+; GFX6-NEXT: v_rcp_f32_e32 v5, v3
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
+; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: s_and_b32 s4, s11, 0xffff
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
-; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v4
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
; GFX6-NEXT: s_and_b32 s4, s9, 0xffff
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
+; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v2, vcc
+; GFX6-NEXT: v_mad_f32 v2, -v1, v3, v4
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
+; GFX6-NEXT: s_and_b32 s4, s7, 0xffff
; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4
-; GFX6-NEXT: v_rcp_f32_e32 v6, v4
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_rcp_f32_e32 v6, v4
+; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
+; GFX6-NEXT: s_mov_b32 s1, s5
+; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX6-NEXT: v_mul_f32_e32 v2, v5, v6
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
-; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX6-NEXT: v_mad_f32 v2, -v2, v4, v5
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -3881,48 +3884,47 @@ define amdgpu_kernel void @udiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
;
; GFX9-LABEL: udiv_v3i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v6, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s7, s2, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_and_b32 s6, s0, 0xffff
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s6
+; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s6, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
+; GFX9-NEXT: s_lshr_b32 s6, s6, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s6
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s4
; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s2
; GFX9-NEXT: v_rcp_f32_e32 v5, v1
; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: s_and_b32 s0, s3, 0xffff
+; GFX9-NEXT: s_and_b32 s2, s7, 0xffff
; GFX9-NEXT: v_cvt_u32_f32_e32 v7, v4
; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v4, s2
; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
; GFX9-NEXT: v_trunc_f32_e32 v2, v5
-; GFX9-NEXT: s_and_b32 s0, s1, 0xffff
+; GFX9-NEXT: s_and_b32 s2, s3, 0xffff
; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v7, vcc
; GFX9-NEXT: v_mad_f32 v3, -v2, v1, v3
; GFX9-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s2
; GFX9-NEXT: v_rcp_f32_e32 v7, v4
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v1
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v2, vcc
; GFX9-NEXT: v_mul_f32_e32 v2, v5, v7
; GFX9-NEXT: v_trunc_f32_e32 v2, v2
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
; GFX9-NEXT: v_cvt_u32_f32_e32 v3, v2
; GFX9-NEXT: v_mad_f32 v2, -v2, v4, v5
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v3, vcc
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_short v6, v2, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v6, v0, s[6:7]
+; GFX9-NEXT: global_store_short v6, v2, s[0:1] offset:4
+; GFX9-NEXT: global_store_dword v6, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = udiv <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -4002,52 +4004,54 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
;
; GFX6-LABEL: urem_v3i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_and_b32 s5, s10, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_lshr_b32 s5, s10, 16
-; GFX6-NEXT: s_and_b32 s4, s8, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s5
-; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s4
-; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_lshr_b32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s4
-; GFX6-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-NEXT: v_mul_f32_e32 v3, v1, v3
-; GFX6-NEXT: v_trunc_f32_e32 v3, v3
-; GFX6-NEXT: v_mad_f32 v1, -v3, v0, v1
-; GFX6-NEXT: v_cvt_u32_f32_e32 v6, v3
+; GFX6-NEXT: s_lshr_b32 s6, s10, 16
+; GFX6-NEXT: s_and_b32 s1, s4, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s1
+; GFX6-NEXT: s_and_b32 s1, s10, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v1, s1
+; GFX6-NEXT: s_lshr_b32 s7, s4, 16
+; GFX6-NEXT: v_rcp_f32_e32 v2, v0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s7
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s6
+; GFX6-NEXT: s_mov_b32 s0, s8
+; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
+; GFX6-NEXT: v_trunc_f32_e32 v2, v2
+; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
+; GFX6-NEXT: v_cvt_u32_f32_e32 v5, v2
; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v1|, v0
-; GFX6-NEXT: v_mul_f32_e32 v1, v4, v5
+; GFX6-NEXT: v_rcp_f32_e32 v1, v4
+; GFX6-NEXT: s_mov_b32 s1, s9
+; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v5, vcc
+; GFX6-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s4
; GFX6-NEXT: v_trunc_f32_e32 v1, v1
-; GFX6-NEXT: s_and_b32 s6, s11, 0xffff
-; GFX6-NEXT: v_mad_f32 v3, -v1, v2, v4
-; GFX6-NEXT: v_cvt_f32_u32_e32 v4, s6
-; GFX6-NEXT: v_addc_u32_e32 v0, vcc, 0, v6, vcc
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
-; GFX6-NEXT: s_and_b32 s6, s9, 0xffff
-; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s6
-; GFX6-NEXT: v_rcp_f32_e32 v6, v4
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
+; GFX6-NEXT: s_and_b32 s4, s5, 0xffff
+; GFX6-NEXT: v_mad_f32 v2, -v1, v4, v3
+; GFX6-NEXT: v_cvt_f32_u32_e32 v3, s4
+; GFX6-NEXT: s_and_b32 s4, s11, 0xffff
+; GFX6-NEXT: v_cvt_f32_u32_e32 v5, s4
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s10, v0
+; GFX6-NEXT: v_rcp_f32_e32 v6, v3
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v3|, v2
+; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
+; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_mul_f32_e32 v2, v5, v6
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
-; GFX6-NEXT: v_cvt_u32_f32_e32 v3, v2
+; GFX6-NEXT: v_cvt_u32_f32_e32 v4, v2
; GFX6-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; GFX6-NEXT: v_mad_f32 v2, -v2, v4, v5
-; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v4
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s5
-; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
-; GFX6-NEXT: v_mul_lo_u32 v2, v2, s11
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s4, v1
+; GFX6-NEXT: v_mad_f32 v2, -v2, v3, v5
+; GFX6-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s7
+; GFX6-NEXT: v_addc_u32_e32 v2, vcc, 0, v4, vcc
+; GFX6-NEXT: v_mul_lo_u32 v2, v2, s5
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s6, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s9, v2
-; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s11, v2
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -4055,33 +4059,34 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
;
; GFX9-LABEL: urem_v3i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s9, s2, 0xffff
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s9
-; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s2
-; GFX9-NEXT: s_and_b32 s8, s0, 0xffff
-; GFX9-NEXT: s_lshr_b32 s0, s0, 16
-; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s8
+; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX9-NEXT: s_and_b32 s5, s6, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s5
+; GFX9-NEXT: v_cvt_f32_u32_e32 v2, s4
+; GFX9-NEXT: s_lshr_b32 s6, s6, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s6
; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s0
+; GFX9-NEXT: s_lshr_b32 s2, s2, 16
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s2
; GFX9-NEXT: v_rcp_f32_e32 v5, v1
-; GFX9-NEXT: s_and_b32 s3, s3, 0xffff
; GFX9-NEXT: v_mul_f32_e32 v4, v2, v4
; GFX9-NEXT: v_trunc_f32_e32 v4, v4
-; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
-; GFX9-NEXT: v_trunc_f32_e32 v5, v5
-; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
; GFX9-NEXT: v_cvt_u32_f32_e32 v6, v4
+; GFX9-NEXT: v_mad_f32 v2, -v4, v0, v2
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v0
-; GFX9-NEXT: v_mad_f32 v2, -v5, v1, v3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s3
-; GFX9-NEXT: s_and_b32 s1, s1, 0xffff
+; GFX9-NEXT: v_mul_f32_e32 v5, v3, v5
; GFX9-NEXT: v_addc_co_u32_e32 v0, vcc, 0, v6, vcc
+; GFX9-NEXT: v_trunc_f32_e32 v5, v5
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s5
+; GFX9-NEXT: s_and_b32 s5, s7, 0xffff
+; GFX9-NEXT: v_mad_f32 v2, -v5, v1, v3
+; GFX9-NEXT: v_cvt_f32_u32_e32 v3, s5
+; GFX9-NEXT: s_and_b32 s3, s3, 0xffff
; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v5
-; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s1
+; GFX9-NEXT: v_cvt_f32_u32_e32 v5, s3
; GFX9-NEXT: v_rcp_f32_e32 v6, v3
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v1
; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v4, vcc
@@ -4090,18 +4095,17 @@ define amdgpu_kernel void @urem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cvt_u32_f32_e32 v4, v2
; GFX9-NEXT: v_mad_f32 v2, -v2, v3, v5
; GFX9-NEXT: v_cmp_ge_f32_e64 vcc, |v2|, v3
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s9
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s6
; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v4, vcc
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s2
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s3
-; GFX9-NEXT: v_sub_u32_e32 v0, s8, v0
+; GFX9-NEXT: v_mul_lo_u32 v2, v2, s5
+; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
; GFX9-NEXT: v_mov_b32_e32 v3, 0
-; GFX9-NEXT: v_sub_u32_e32 v1, s0, v1
-; GFX9-NEXT: v_sub_u32_e32 v2, s1, v2
+; GFX9-NEXT: v_sub_u32_e32 v1, s2, v1
+; GFX9-NEXT: v_sub_u32_e32 v2, s3, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX9-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX9-NEXT: global_store_short v3, v2, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
+; GFX9-NEXT: global_store_short v3, v2, s[0:1] offset:4
+; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = urem <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -4187,46 +4191,47 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
;
; GFX6-LABEL: sdiv_v3i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_sext_i32_i16 s4, s10
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
+; GFX6-NEXT: s_sext_i32_i16 s7, s10
+; GFX6-NEXT: s_sext_i32_i16 s6, s4
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s6
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s7
+; GFX6-NEXT: s_xor_b32 s6, s7, s6
+; GFX6-NEXT: s_ashr_i32 s6, s6, 30
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: s_mov_b32 s0, s8
+; GFX6-NEXT: s_or_b32 s8, s6, 1
+; GFX6-NEXT: s_sext_i32_i16 s5, s5
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[6:7], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[6:7], s[6:7], exec
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: s_ashr_i32 s5, s10, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s8, 16
-; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
+; GFX6-NEXT: s_cselect_b32 s6, s8, 0
+; GFX6-NEXT: s_ashr_i32 s4, s4, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, s6, v2
+; GFX6-NEXT: s_ashr_i32 s6, s10, 16
+; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s6
; GFX6-NEXT: v_rcp_f32_e32 v3, v0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
+; GFX6-NEXT: s_xor_b32 s4, s6, s4
; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: s_or_b32 s4, s4, 1
; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3
; GFX6-NEXT: v_trunc_f32_e32 v3, v3
; GFX6-NEXT: v_mad_f32 v2, -v3, v0, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: s_sext_i32_i16 s5, s11
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[6:7], |v2|, |v0|
; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s5
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
+; GFX6-NEXT: s_and_b64 s[6:7], s[6:7], exec
+; GFX6-NEXT: s_cselect_b32 s4, s4, 0
; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v3
-; GFX6-NEXT: s_sext_i32_i16 s4, s9
+; GFX6-NEXT: s_sext_i32_i16 s4, s11
; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s4
; GFX6-NEXT: v_rcp_f32_e32 v4, v0
; GFX6-NEXT: s_xor_b32 s4, s4, s5
@@ -4239,6 +4244,7 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX6-NEXT: s_cselect_b32 s4, s6, 0
+; GFX6-NEXT: s_mov_b32 s1, s9
; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v4
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_and_b32_e32 v1, 0xffff, v1
@@ -4249,13 +4255,13 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
;
; GFX9-LABEL: sdiv_v3i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s4, s2
+; GFX9-NEXT: s_sext_i32_i16 s4, s6
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX9-NEXT: s_sext_i32_i16 s5, s0
+; GFX9-NEXT: s_sext_i32_i16 s5, s2
; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s5
; GFX9-NEXT: s_xor_b32 s4, s5, s4
; GFX9-NEXT: v_rcp_f32_e32 v3, v0
@@ -4267,44 +4273,44 @@ define amdgpu_kernel void @sdiv_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX9-NEXT: s_cselect_b32 s4, s8, 0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 16
+; GFX9-NEXT: s_ashr_i32 s5, s6, 16
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 16
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s5
+; GFX9-NEXT: s_ashr_i32 s2, s2, 16
; GFX9-NEXT: v_add_u32_e32 v2, s4, v3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s0
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s2
; GFX9-NEXT: v_rcp_f32_e32 v4, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_sext_i32_i16 s2, s3
+; GFX9-NEXT: s_xor_b32 s2, s2, s5
+; GFX9-NEXT: s_ashr_i32 s2, s2, 30
+; GFX9-NEXT: s_or_b32 s2, s2, 1
; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4
; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_mad_f32 v3, -v4, v0, v3
-; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: s_or_b32 s0, s0, 1
; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v0|
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: v_add_u32_e32 v3, s0, v4
-; GFX9-NEXT: s_sext_i32_i16 s0, s1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s0
+; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
+; GFX9-NEXT: s_sext_i32_i16 s4, s7
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s4
+; GFX9-NEXT: s_cselect_b32 s2, s2, 0
+; GFX9-NEXT: v_add_u32_e32 v3, s2, v4
+; GFX9-NEXT: s_sext_i32_i16 s2, s3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v4, s2
; GFX9-NEXT: v_rcp_f32_e32 v5, v0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s2, s0, 1
+; GFX9-NEXT: s_xor_b32 s2, s2, s4
+; GFX9-NEXT: s_ashr_i32 s2, s2, 30
+; GFX9-NEXT: s_or_b32 s4, s2, 1
; GFX9-NEXT: v_mul_f32_e32 v5, v4, v5
; GFX9-NEXT: v_trunc_f32_e32 v5, v5
; GFX9-NEXT: v_mad_f32 v4, -v5, v0, v4
; GFX9-NEXT: v_cvt_i32_f32_e32 v5, v5
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v4|, |v0|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s0, v5
+; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v4|, |v0|
+; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cselect_b32 s2, s4, 0
+; GFX9-NEXT: v_add_u32_e32 v0, s2, v5
; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX9-NEXT: v_lshl_or_b32 v2, v3, 16, v2
-; GFX9-NEXT: global_store_short v1, v0, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v1, v2, s[6:7]
+; GFX9-NEXT: global_store_short v1, v0, s[0:1] offset:4
+; GFX9-NEXT: global_store_dword v1, v2, s[0:1]
; GFX9-NEXT: s_endpgm
%r = sdiv <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -4396,68 +4402,70 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
;
; GFX6-LABEL: srem_v3i16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX6-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_sext_i32_i16 s4, s10
-; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s8
-; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s5
-; GFX6-NEXT: s_xor_b32 s4, s5, s4
+; GFX6-NEXT: s_sext_i32_i16 s0, s8
+; GFX6-NEXT: v_cvt_f32_i32_e32 v0, s0
+; GFX6-NEXT: s_sext_i32_i16 s1, s6
+; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s1
+; GFX6-NEXT: s_xor_b32 s0, s1, s0
; GFX6-NEXT: v_rcp_f32_e32 v2, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: s_ashr_i32 s0, s0, 30
+; GFX6-NEXT: s_or_b32 s10, s0, 1
; GFX6-NEXT: v_mul_f32_e32 v2, v1, v2
; GFX6-NEXT: v_trunc_f32_e32 v2, v2
; GFX6-NEXT: v_mad_f32 v1, -v2, v0, v1
; GFX6-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
-; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s6, 0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s4, v2
-; GFX6-NEXT: s_ashr_i32 s4, s10, 16
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, |v0|
+; GFX6-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GFX6-NEXT: s_cselect_b32 s0, s10, 0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s0, v2
+; GFX6-NEXT: s_mov_b32 s0, s4
+; GFX6-NEXT: s_ashr_i32 s4, s8, 16
; GFX6-NEXT: v_cvt_f32_i32_e32 v1, s4
-; GFX6-NEXT: s_ashr_i32 s5, s8, 16
+; GFX6-NEXT: s_mov_b32 s1, s5
+; GFX6-NEXT: s_ashr_i32 s5, s6, 16
; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s5
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s10
; GFX6-NEXT: v_rcp_f32_e32 v3, v1
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s8
; GFX6-NEXT: s_xor_b32 s4, s5, s4
; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: s_lshr_b32 s6, s8, 16
; GFX6-NEXT: v_mul_f32_e32 v3, v2, v3
; GFX6-NEXT: v_trunc_f32_e32 v3, v3
; GFX6-NEXT: v_mad_f32 v2, -v3, v1, v2
; GFX6-NEXT: v_cvt_i32_f32_e32 v3, v3
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s8, v0
-; GFX6-NEXT: s_lshr_b32 s7, s10, 16
-; GFX6-NEXT: s_or_b32 s8, s4, 1
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
+; GFX6-NEXT: s_lshr_b32 s10, s6, 16
+; GFX6-NEXT: s_lshr_b32 s8, s8, 16
+; GFX6-NEXT: s_or_b32 s6, s4, 1
; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v1|
; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s8, 0
+; GFX6-NEXT: s_cselect_b32 s4, s6, 0
; GFX6-NEXT: v_add_i32_e32 v1, vcc, s4, v3
-; GFX6-NEXT: s_sext_i32_i16 s4, s11
+; GFX6-NEXT: s_sext_i32_i16 s4, s9
; GFX6-NEXT: v_cvt_f32_i32_e32 v2, s4
-; GFX6-NEXT: s_sext_i32_i16 s5, s9
+; GFX6-NEXT: s_sext_i32_i16 s5, s7
; GFX6-NEXT: v_cvt_f32_i32_e32 v3, s5
; GFX6-NEXT: s_xor_b32 s4, s5, s4
; GFX6-NEXT: v_rcp_f32_e32 v4, v2
; GFX6-NEXT: s_ashr_i32 s4, s4, 30
-; GFX6-NEXT: v_mul_lo_u32 v1, v1, s7
-; GFX6-NEXT: s_or_b32 s7, s4, 1
+; GFX6-NEXT: s_or_b32 s6, s4, 1
+; GFX6-NEXT: v_mul_lo_u32 v1, v1, s8
; GFX6-NEXT: v_mul_f32_e32 v4, v3, v4
; GFX6-NEXT: v_trunc_f32_e32 v4, v4
; GFX6-NEXT: v_mad_f32 v3, -v4, v2, v3
; GFX6-NEXT: v_cvt_i32_f32_e32 v4, v4
; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v3|, |v2|
; GFX6-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX6-NEXT: s_cselect_b32 s4, s7, 0
+; GFX6-NEXT: s_cselect_b32 s4, s6, 0
; GFX6-NEXT: v_add_i32_e32 v2, vcc, s4, v4
-; GFX6-NEXT: v_mul_lo_u32 v2, v2, s11
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s6, v1
+; GFX6-NEXT: v_mul_lo_u32 v2, v2, s9
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s10, v1
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s9, v2
+; GFX6-NEXT: v_sub_i32_e32 v2, vcc, s7, v2
; GFX6-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
; GFX6-NEXT: buffer_store_short v2, off, s[0:3], 0 offset:4
@@ -4466,12 +4474,12 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
;
; GFX9-LABEL: srem_v3i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s8, s2
+; GFX9-NEXT: s_sext_i32_i16 s8, s6
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s8
-; GFX9-NEXT: s_sext_i32_i16 s9, s0
+; GFX9-NEXT: s_sext_i32_i16 s9, s2
; GFX9-NEXT: v_cvt_f32_i32_e32 v1, s9
; GFX9-NEXT: s_xor_b32 s4, s9, s8
; GFX9-NEXT: v_rcp_f32_e32 v2, v0
@@ -4483,51 +4491,51 @@ define amdgpu_kernel void @srem_v3i16(ptr addrspace(1) %out, <3 x i16> %x, <3 x
; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, |v0|
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
; GFX9-NEXT: s_cselect_b32 s4, s10, 0
-; GFX9-NEXT: s_ashr_i32 s10, s0, 16
-; GFX9-NEXT: s_ashr_i32 s0, s2, 16
+; GFX9-NEXT: s_ashr_i32 s10, s2, 16
+; GFX9-NEXT: s_ashr_i32 s2, s6, 16
; GFX9-NEXT: v_cvt_i32_f32_e32 v2, v2
-; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9-NEXT: s_xor_b32 s2, s10, s0
-; GFX9-NEXT: s_ashr_i32 s2, s2, 30
+; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
; GFX9-NEXT: v_add_u32_e32 v1, s4, v2
; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s10
; GFX9-NEXT: v_rcp_f32_e32 v3, v0
-; GFX9-NEXT: s_or_b32 s2, s2, 1
-; GFX9-NEXT: v_mul_lo_u32 v1, v1, s8
+; GFX9-NEXT: s_xor_b32 s4, s10, s2
+; GFX9-NEXT: s_ashr_i32 s4, s4, 30
+; GFX9-NEXT: s_or_b32 s6, s4, 1
; GFX9-NEXT: v_mul_f32_e32 v3, v2, v3
; GFX9-NEXT: v_trunc_f32_e32 v3, v3
; GFX9-NEXT: v_mad_f32 v2, -v3, v0, v2
; GFX9-NEXT: v_cvt_i32_f32_e32 v3, v3
; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, |v0|
; GFX9-NEXT: s_and_b64 s[4:5], s[4:5], exec
-; GFX9-NEXT: s_cselect_b32 s2, s2, 0
-; GFX9-NEXT: v_add_u32_e32 v0, s2, v3
-; GFX9-NEXT: s_sext_i32_i16 s2, s3
-; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s2
-; GFX9-NEXT: s_sext_i32_i16 s3, s1
-; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s3
-; GFX9-NEXT: v_mul_lo_u32 v0, v0, s0
+; GFX9-NEXT: s_cselect_b32 s4, s6, 0
+; GFX9-NEXT: v_add_u32_e32 v0, s4, v3
+; GFX9-NEXT: s_sext_i32_i16 s4, s7
+; GFX9-NEXT: v_cvt_f32_i32_e32 v2, s4
+; GFX9-NEXT: s_sext_i32_i16 s5, s3
+; GFX9-NEXT: v_cvt_f32_i32_e32 v3, s5
+; GFX9-NEXT: v_mul_lo_u32 v0, v0, s2
; GFX9-NEXT: v_rcp_f32_e32 v4, v2
-; GFX9-NEXT: s_xor_b32 s0, s3, s2
-; GFX9-NEXT: s_ashr_i32 s0, s0, 30
-; GFX9-NEXT: s_or_b32 s4, s0, 1
+; GFX9-NEXT: s_xor_b32 s2, s5, s4
+; GFX9-NEXT: s_ashr_i32 s2, s2, 30
+; GFX9-NEXT: s_or_b32 s6, s2, 1
; GFX9-NEXT: v_mul_f32_e32 v4, v3, v4
; GFX9-NEXT: v_trunc_f32_e32 v4, v4
; GFX9-NEXT: v_mad_f32 v3, -v4, v2, v3
; GFX9-NEXT: v_cvt_i32_f32_e32 v4, v4
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v3|, |v2|
-; GFX9-NEXT: s_and_b64 s[0:1], s[0:1], exec
-; GFX9-NEXT: s_cselect_b32 s0, s4, 0
-; GFX9-NEXT: v_add_u32_e32 v2, s0, v4
-; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
-; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
+; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v3|, |v2|
+; GFX9-NEXT: s_and_b64 s[2:3], s[2:3], exec
+; GFX9-NEXT: s_cselect_b32 s2, s6, 0
+; GFX9-NEXT: v_mul_lo_u32 v1, v1, s8
+; GFX9-NEXT: v_add_u32_e32 v2, s2, v4
+; GFX9-NEXT: v_mul_lo_u32 v2, v2, s4
; GFX9-NEXT: v_mov_b32_e32 v3, 0
+; GFX9-NEXT: v_sub_u32_e32 v1, s9, v1
; GFX9-NEXT: v_sub_u32_e32 v0, s10, v0
-; GFX9-NEXT: v_sub_u32_e32 v2, s3, v2
+; GFX9-NEXT: v_sub_u32_e32 v2, s5, v2
; GFX9-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX9-NEXT: v_lshl_or_b32 v0, v0, 16, v1
-; GFX9-NEXT: global_store_short v3, v2, s[6:7] offset:4
-; GFX9-NEXT: global_store_dword v3, v0, s[6:7]
+; GFX9-NEXT: global_store_short v3, v2, s[0:1] offset:4
+; GFX9-NEXT: global_store_dword v3, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%r = srem <3 x i16> %x, %y
store <3 x i16> %r, ptr addrspace(1) %out
@@ -5382,24 +5390,24 @@ define amdgpu_kernel void @udiv_i32_oddk_denom(ptr addrspace(1) %out, i32 %x) {
;
; GFX6-LABEL: udiv_i32_oddk_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: v_mov_b32_e32 v0, 0xb2a50881
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_hi_u32 v0, s6, v0
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s6, v0
+; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s2, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 1, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v1, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 20, v0
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: udiv_i32_oddk_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_mul_hi_u32 s3, s2, 0xb2a50881
@@ -5423,20 +5431,20 @@ define amdgpu_kernel void @udiv_i32_pow2k_denom(ptr addrspace(1) %out, i32 %x) {
;
; GFX6-LABEL: udiv_i32_pow2k_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_lshr_b32 s4, s6, 12
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_lshr_b32 s0, s2, 12
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: udiv_i32_pow2k_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s2, s2, 12
@@ -5458,14 +5466,15 @@ define amdgpu_kernel void @udiv_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
; GFX6-LABEL: udiv_i32_pow2_shl_denom:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_add_i32 s5, s5, 12
-; GFX6-NEXT: s_lshr_b32 s4, s4, s5
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_add_i32 s0, s3, 12
+; GFX6-NEXT: s_lshr_b32 s0, s2, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: udiv_i32_pow2_shl_denom:
@@ -5498,15 +5507,16 @@ define amdgpu_kernel void @udiv_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
; GFX6-LABEL: udiv_v2i32_pow2k_denom:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_lshr_b32 s4, s4, 12
-; GFX6-NEXT: s_lshr_b32 s5, s5, 12
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: v_mov_b32_e32 v1, s5
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_lshr_b32 s0, s2, 12
+; GFX6-NEXT: s_lshr_b32 s1, s3, 12
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: udiv_v2i32_pow2k_denom:
@@ -5540,18 +5550,19 @@ define amdgpu_kernel void @udiv_v2i32_mixed_pow2k_denom(ptr addrspace(1) %out, <
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: v_mov_b32_e32 v0, 0x100101
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: v_mul_hi_u32 v0, s5, v0
-; GFX6-NEXT: s_lshr_b32 s4, s4, 12
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s5, v0
+; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_lshr_b32 s0, s2, 12
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s3, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 1, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v1, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 11, v0
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: udiv_v2i32_mixed_pow2k_denom:
@@ -5646,29 +5657,31 @@ define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
;
; GFX6-LABEL: udiv_v2i32_pow2_shl_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: s_mov_b32 s6, -1
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; GFX6-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s11, 0xf000
+; GFX6-NEXT: s_mov_b32 s10, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_lshl_b32 s0, 0x1000, s10
+; GFX6-NEXT: s_lshl_b32 s0, 0x1000, s0
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s0
-; GFX6-NEXT: s_sub_i32 s1, 0, s0
-; GFX6-NEXT: s_lshl_b32 s2, 0x1000, s11
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s2
+; GFX6-NEXT: s_sub_i32 s2, 0, s0
+; GFX6-NEXT: s_mov_b32 s8, s4
+; GFX6-NEXT: s_mov_b32 s9, s5
; GFX6-NEXT: v_rcp_f32_e32 v0, v0
-; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s1, v0
+; GFX6-NEXT: v_mul_lo_u32 v1, s2, v0
+; GFX6-NEXT: s_lshl_b32 s2, 0x1000, s1
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s2
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s8, v0
+; GFX6-NEXT: v_mul_hi_u32 v0, s6, v0
; GFX6-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v2
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX6-NEXT: v_readfirstlane_b32 s1, v0
; GFX6-NEXT: s_mul_i32 s1, s1, s0
-; GFX6-NEXT: s_sub_i32 s1, s8, s1
+; GFX6-NEXT: s_sub_i32 s1, s6, s1
; GFX6-NEXT: s_sub_i32 s3, s1, s0
; GFX6-NEXT: s_cmp_ge_u32 s1, s0
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v0
@@ -5683,10 +5696,10 @@ define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX6-NEXT: v_mul_hi_u32 v3, v1, v3
; GFX6-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[0:1]
; GFX6-NEXT: v_add_i32_e32 v1, vcc, v1, v3
-; GFX6-NEXT: v_mul_hi_u32 v1, s9, v1
+; GFX6-NEXT: v_mul_hi_u32 v1, s7, v1
; GFX6-NEXT: v_readfirstlane_b32 s0, v1
; GFX6-NEXT: s_mul_i32 s0, s0, s2
-; GFX6-NEXT: s_sub_i32 s0, s9, s0
+; GFX6-NEXT: s_sub_i32 s0, s7, s0
; GFX6-NEXT: s_sub_i32 s1, s0, s2
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v1
; GFX6-NEXT: s_cmp_ge_u32 s0, s2
@@ -5697,19 +5710,19 @@ define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX6-NEXT: s_cmp_ge_u32 s0, s2
; GFX6-NEXT: s_cselect_b64 vcc, -1, 0
; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: udiv_v2i32_pow2_shl_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s7, 0x1000, s2
+; GFX9-NEXT: s_lshl_b32 s7, 0x1000, s0
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_lshl_b32 s6, 0x1000, s3
+; GFX9-NEXT: s_lshl_b32 s6, 0x1000, s1
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s6
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_sub_i32 s4, 0, s7
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
@@ -5721,37 +5734,37 @@ define amdgpu_kernel void @udiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX9-NEXT: s_mul_i32 s4, s4, s5
; GFX9-NEXT: s_mul_hi_u32 s4, s5, s4
; GFX9-NEXT: s_add_i32 s5, s5, s4
-; GFX9-NEXT: s_mul_hi_u32 s4, s0, s5
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mul_hi_u32 s4, s2, s5
; GFX9-NEXT: s_mul_i32 s5, s4, s7
-; GFX9-NEXT: s_sub_i32 s0, s0, s5
+; GFX9-NEXT: s_sub_i32 s2, s2, s5
; GFX9-NEXT: s_add_i32 s9, s4, 1
-; GFX9-NEXT: s_sub_i32 s5, s0, s7
-; GFX9-NEXT: s_cmp_ge_u32 s0, s7
+; GFX9-NEXT: s_sub_i32 s5, s2, s7
+; GFX9-NEXT: s_cmp_ge_u32 s2, s7
; GFX9-NEXT: s_cselect_b32 s4, s9, s4
-; GFX9-NEXT: s_cselect_b32 s0, s5, s0
+; GFX9-NEXT: s_cselect_b32 s2, s5, s2
; GFX9-NEXT: s_add_i32 s5, s4, 1
-; GFX9-NEXT: s_cmp_ge_u32 s0, s7
+; GFX9-NEXT: s_cmp_ge_u32 s2, s7
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
-; GFX9-NEXT: s_cselect_b32 s0, s5, s4
+; GFX9-NEXT: s_cselect_b32 s2, s5, s4
; GFX9-NEXT: s_sub_i32 s4, 0, s6
; GFX9-NEXT: s_mul_i32 s4, s4, s8
; GFX9-NEXT: s_mul_hi_u32 s4, s8, s4
; GFX9-NEXT: s_add_i32 s8, s8, s4
-; GFX9-NEXT: s_mul_hi_u32 s4, s1, s8
+; GFX9-NEXT: s_mul_hi_u32 s4, s3, s8
; GFX9-NEXT: s_mul_i32 s5, s4, s6
-; GFX9-NEXT: s_sub_i32 s1, s1, s5
+; GFX9-NEXT: s_sub_i32 s3, s3, s5
; GFX9-NEXT: s_add_i32 s7, s4, 1
-; GFX9-NEXT: s_sub_i32 s5, s1, s6
-; GFX9-NEXT: s_cmp_ge_u32 s1, s6
+; GFX9-NEXT: s_sub_i32 s5, s3, s6
+; GFX9-NEXT: s_cmp_ge_u32 s3, s6
; GFX9-NEXT: s_cselect_b32 s4, s7, s4
-; GFX9-NEXT: s_cselect_b32 s1, s5, s1
+; GFX9-NEXT: s_cselect_b32 s3, s5, s3
; GFX9-NEXT: s_add_i32 s5, s4, 1
-; GFX9-NEXT: s_cmp_ge_u32 s1, s6
-; GFX9-NEXT: s_cselect_b32 s1, s5, s4
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT: s_cmp_ge_u32 s3, s6
+; GFX9-NEXT: s_cselect_b32 s3, s5, s4
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
%shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
%r = udiv <2 x i32> %x, %shl.y
@@ -5767,27 +5780,27 @@ define amdgpu_kernel void @urem_i32_oddk_denom(ptr addrspace(1) %out, i32 %x) {
;
; GFX6-LABEL: urem_i32_oddk_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: v_mov_b32_e32 v0, 0xb2a50881
-; GFX6-NEXT: s_mov_b32 s2, 0x12d8fb
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_hi_u32 v0, s6, v0
-; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s6, v0
+; GFX6-NEXT: s_mov_b32 s3, 0x12d8fb
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
+; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_sub_i32_e32 v1, vcc, s2, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 1, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v1, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v0, 20, v0
-; GFX6-NEXT: v_mul_lo_u32 v0, v0, s2
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: v_mul_lo_u32 v0, v0, s3
+; GFX6-NEXT: v_sub_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: urem_i32_oddk_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_mul_hi_u32 s3, s2, 0xb2a50881
@@ -5813,20 +5826,20 @@ define amdgpu_kernel void @urem_i32_pow2k_denom(ptr addrspace(1) %out, i32 %x) {
;
; GFX6-LABEL: urem_i32_pow2k_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_and_b32 s4, s6, 0xfff
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_and_b32 s0, s2, 0xfff
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: urem_i32_pow2k_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0xfff
@@ -5848,15 +5861,16 @@ define amdgpu_kernel void @urem_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
; GFX6-LABEL: urem_i32_pow2_shl_denom:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_lshl_b32 s5, 0x1000, s5
-; GFX6-NEXT: s_add_i32 s5, s5, -1
-; GFX6-NEXT: s_and_b32 s4, s4, s5
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_lshl_b32 s0, 0x1000, s3
+; GFX6-NEXT: s_add_i32 s0, s0, -1
+; GFX6-NEXT: s_and_b32 s0, s2, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: urem_i32_pow2_shl_denom:
@@ -5890,15 +5904,16 @@ define amdgpu_kernel void @urem_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
; GFX6-LABEL: urem_v2i32_pow2k_denom:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_and_b32 s4, s4, 0xfff
-; GFX6-NEXT: s_and_b32 s5, s5, 0xfff
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: v_mov_b32_e32 v1, s5
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_and_b32 s0, s2, 0xfff
+; GFX6-NEXT: s_and_b32 s1, s3, 0xfff
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: urem_v2i32_pow2k_denom:
@@ -5985,64 +6000,67 @@ define amdgpu_kernel void @urem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
;
; GFX6-LABEL: urem_v2i32_pow2_shl_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_lshl_b32 s2, 0x1000, s2
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX6-NEXT: s_sub_i32 s6, 0, s2
-; GFX6-NEXT: s_lshl_b32 s3, 0x1000, s3
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s3
+; GFX6-NEXT: s_lshl_b32 s6, 0x1000, s0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6
+; GFX6-NEXT: s_sub_i32 s0, 0, s6
+; GFX6-NEXT: s_lshl_b32 s8, 0x1000, s1
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s8
; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v1, s0, v0
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
; GFX6-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v2
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_readfirstlane_b32 s6, v0
-; GFX6-NEXT: s_mul_i32 s6, s6, s2
-; GFX6-NEXT: s_sub_i32 s0, s0, s6
-; GFX6-NEXT: s_sub_i32 s6, s0, s2
-; GFX6-NEXT: s_cmp_ge_u32 s0, s2
-; GFX6-NEXT: s_cselect_b32 s0, s6, s0
-; GFX6-NEXT: s_sub_i32 s6, s0, s2
-; GFX6-NEXT: s_cmp_ge_u32 s0, s2
-; GFX6-NEXT: s_cselect_b32 s0, s6, s0
-; GFX6-NEXT: s_sub_i32 s2, 0, s3
-; GFX6-NEXT: v_mul_lo_u32 v0, s2, v1
+; GFX6-NEXT: v_readfirstlane_b32 s4, v0
+; GFX6-NEXT: s_mul_i32 s4, s4, s6
+; GFX6-NEXT: s_sub_i32 s2, s2, s4
+; GFX6-NEXT: s_sub_i32 s4, s2, s6
+; GFX6-NEXT: s_cmp_ge_u32 s2, s6
+; GFX6-NEXT: s_cselect_b32 s2, s4, s2
+; GFX6-NEXT: s_sub_i32 s4, s2, s6
+; GFX6-NEXT: s_cmp_ge_u32 s2, s6
+; GFX6-NEXT: s_cselect_b32 s2, s4, s2
+; GFX6-NEXT: s_sub_i32 s4, 0, s8
+; GFX6-NEXT: v_mul_lo_u32 v0, s4, v1
+; GFX6-NEXT: s_mov_b32 s4, s0
; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: v_mul_hi_u32 v0, v1, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; GFX6-NEXT: v_mul_hi_u32 v0, s1, v0
-; GFX6-NEXT: v_readfirstlane_b32 s2, v0
-; GFX6-NEXT: s_mul_i32 s2, s2, s3
-; GFX6-NEXT: s_sub_i32 s1, s1, s2
-; GFX6-NEXT: s_sub_i32 s2, s1, s3
-; GFX6-NEXT: s_cmp_ge_u32 s1, s3
-; GFX6-NEXT: s_cselect_b32 s1, s2, s1
-; GFX6-NEXT: s_sub_i32 s2, s1, s3
-; GFX6-NEXT: s_cmp_ge_u32 s1, s3
-; GFX6-NEXT: s_cselect_b32 s1, s2, s1
-; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_mul_i32 s0, s0, s8
+; GFX6-NEXT: s_sub_i32 s0, s3, s0
+; GFX6-NEXT: s_sub_i32 s1, s0, s8
+; GFX6-NEXT: s_cmp_ge_u32 s0, s8
+; GFX6-NEXT: s_cselect_b32 s0, s1, s0
+; GFX6-NEXT: s_sub_i32 s1, s0, s8
+; GFX6-NEXT: s_cmp_ge_u32 s0, s8
+; GFX6-NEXT: s_cselect_b32 s0, s1, s0
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: urem_v2i32_pow2_shl_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s7, 0x1000, s2
+; GFX9-NEXT: s_lshl_b32 s7, 0x1000, s0
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_lshl_b32 s6, 0x1000, s3
+; GFX9-NEXT: s_lshl_b32 s6, 0x1000, s1
; GFX9-NEXT: v_cvt_f32_u32_e32 v1, s6
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: s_sub_i32 s4, 0, s7
; GFX9-NEXT: v_rcp_f32_e32 v1, v1
@@ -6054,33 +6072,33 @@ define amdgpu_kernel void @urem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
; GFX9-NEXT: s_mul_i32 s4, s4, s5
; GFX9-NEXT: s_mul_hi_u32 s4, s5, s4
; GFX9-NEXT: s_add_i32 s5, s5, s4
-; GFX9-NEXT: s_mul_hi_u32 s4, s0, s5
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mul_hi_u32 s4, s2, s5
; GFX9-NEXT: s_mul_i32 s4, s4, s7
-; GFX9-NEXT: s_sub_i32 s0, s0, s4
-; GFX9-NEXT: s_sub_i32 s4, s0, s7
-; GFX9-NEXT: s_cmp_ge_u32 s0, s7
-; GFX9-NEXT: s_cselect_b32 s0, s4, s0
-; GFX9-NEXT: s_sub_i32 s4, s0, s7
-; GFX9-NEXT: s_cmp_ge_u32 s0, s7
+; GFX9-NEXT: s_sub_i32 s2, s2, s4
+; GFX9-NEXT: s_sub_i32 s4, s2, s7
+; GFX9-NEXT: s_cmp_ge_u32 s2, s7
+; GFX9-NEXT: s_cselect_b32 s2, s4, s2
+; GFX9-NEXT: s_sub_i32 s4, s2, s7
+; GFX9-NEXT: s_cmp_ge_u32 s2, s7
; GFX9-NEXT: v_readfirstlane_b32 s8, v1
-; GFX9-NEXT: s_cselect_b32 s0, s4, s0
+; GFX9-NEXT: s_cselect_b32 s2, s4, s2
; GFX9-NEXT: s_sub_i32 s4, 0, s6
; GFX9-NEXT: s_mul_i32 s4, s4, s8
; GFX9-NEXT: s_mul_hi_u32 s4, s8, s4
; GFX9-NEXT: s_add_i32 s8, s8, s4
-; GFX9-NEXT: s_mul_hi_u32 s4, s1, s8
+; GFX9-NEXT: s_mul_hi_u32 s4, s3, s8
; GFX9-NEXT: s_mul_i32 s4, s4, s6
-; GFX9-NEXT: s_sub_i32 s1, s1, s4
-; GFX9-NEXT: s_sub_i32 s4, s1, s6
-; GFX9-NEXT: s_cmp_ge_u32 s1, s6
-; GFX9-NEXT: s_cselect_b32 s1, s4, s1
-; GFX9-NEXT: s_sub_i32 s4, s1, s6
-; GFX9-NEXT: s_cmp_ge_u32 s1, s6
-; GFX9-NEXT: s_cselect_b32 s1, s4, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT: s_sub_i32 s3, s3, s4
+; GFX9-NEXT: s_sub_i32 s4, s3, s6
+; GFX9-NEXT: s_cmp_ge_u32 s3, s6
+; GFX9-NEXT: s_cselect_b32 s3, s4, s3
+; GFX9-NEXT: s_sub_i32 s4, s3, s6
+; GFX9-NEXT: s_cmp_ge_u32 s3, s6
+; GFX9-NEXT: s_cselect_b32 s3, s4, s3
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
%shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
%r = urem <2 x i32> %x, %shl.y
@@ -6096,24 +6114,24 @@ define amdgpu_kernel void @sdiv_i32_oddk_denom(ptr addrspace(1) %out, i32 %x) {
;
; GFX6-LABEL: sdiv_i32_oddk_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: v_mov_b32_e32 v0, 0xd9528441
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_hi_i32 v0, s6, v0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; GFX6-NEXT: v_mul_hi_i32 v0, s2, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s2, v0
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 31, v0
; GFX6-NEXT: v_ashrrev_i32_e32 v0, 20, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: sdiv_i32_oddk_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_mul_hi_i32 s3, s2, 0xd9528441
@@ -6137,23 +6155,23 @@ define amdgpu_kernel void @sdiv_i32_pow2k_denom(ptr addrspace(1) %out, i32 %x) {
;
; GFX6-LABEL: sdiv_i32_pow2k_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_ashr_i32 s4, s6, 31
-; GFX6-NEXT: s_lshr_b32 s4, s4, 20
-; GFX6-NEXT: s_add_i32 s6, s6, s4
-; GFX6-NEXT: s_ashr_i32 s4, s6, 12
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_ashr_i32 s0, s2, 31
+; GFX6-NEXT: s_lshr_b32 s0, s0, 20
+; GFX6-NEXT: s_add_i32 s0, s2, s0
+; GFX6-NEXT: s_ashr_i32 s0, s0, 12
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: sdiv_i32_pow2k_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_ashr_i32 s3, s2, 31
@@ -6178,40 +6196,41 @@ define amdgpu_kernel void @sdiv_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
; GFX6-LABEL: sdiv_i32_pow2_shl_denom:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_lshl_b32 s5, 0x1000, s5
-; GFX6-NEXT: s_abs_i32 s6, s5
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX6-NEXT: s_sub_i32 s2, 0, s6
-; GFX6-NEXT: s_abs_i32 s7, s4
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_lshl_b32 s3, 0x1000, s3
+; GFX6-NEXT: s_abs_i32 s8, s3
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s8
+; GFX6-NEXT: s_sub_i32 s4, 0, s8
+; GFX6-NEXT: s_abs_i32 s9, s2
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s2, v0
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: v_mul_lo_u32 v1, s4, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s7, v0
-; GFX6-NEXT: v_readfirstlane_b32 s8, v0
-; GFX6-NEXT: s_mul_i32 s8, s8, s6
-; GFX6-NEXT: s_sub_i32 s7, s7, s8
-; GFX6-NEXT: s_sub_i32 s8, s7, s6
+; GFX6-NEXT: v_mul_hi_u32 v0, s9, v0
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_mul_i32 s0, s0, s8
+; GFX6-NEXT: s_sub_i32 s0, s9, s0
+; GFX6-NEXT: s_sub_i32 s1, s0, s8
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT: s_cmp_ge_u32 s7, s6
+; GFX6-NEXT: s_cmp_ge_u32 s0, s8
; GFX6-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX6-NEXT: s_cselect_b32 s7, s8, s7
+; GFX6-NEXT: s_cselect_b32 s0, s1, s0
; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT: s_cmp_ge_u32 s7, s6
+; GFX6-NEXT: s_cmp_ge_u32 s0, s8
; GFX6-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX6-NEXT: s_xor_b32 s4, s4, s5
+; GFX6-NEXT: s_xor_b32 s0, s2, s3
; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT: s_ashr_i32 s4, s4, 31
-; GFX6-NEXT: v_xor_b32_e32 v0, s4, v0
-; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s4, v0
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_ashr_i32 s0, s0, 31
+; GFX6-NEXT: v_xor_b32_e32 v0, s0, v0
+; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: sdiv_i32_pow2_shl_denom:
@@ -6269,21 +6288,22 @@ define amdgpu_kernel void @sdiv_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
; GFX6-LABEL: sdiv_v2i32_pow2k_denom:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_ashr_i32 s6, s4, 31
-; GFX6-NEXT: s_ashr_i32 s7, s5, 31
-; GFX6-NEXT: s_lshr_b32 s6, s6, 20
-; GFX6-NEXT: s_lshr_b32 s7, s7, 20
-; GFX6-NEXT: s_add_i32 s4, s4, s6
-; GFX6-NEXT: s_add_i32 s5, s5, s7
-; GFX6-NEXT: s_ashr_i32 s4, s4, 12
-; GFX6-NEXT: s_ashr_i32 s5, s5, 12
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: v_mov_b32_e32 v1, s5
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_ashr_i32 s0, s2, 31
+; GFX6-NEXT: s_ashr_i32 s1, s3, 31
+; GFX6-NEXT: s_lshr_b32 s0, s0, 20
+; GFX6-NEXT: s_lshr_b32 s1, s1, 20
+; GFX6-NEXT: s_add_i32 s0, s2, s0
+; GFX6-NEXT: s_add_i32 s1, s3, s1
+; GFX6-NEXT: s_ashr_i32 s0, s0, 12
+; GFX6-NEXT: s_ashr_i32 s1, s1, 12
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: sdiv_v2i32_pow2k_denom:
@@ -6323,21 +6343,22 @@ define amdgpu_kernel void @ssdiv_v2i32_mixed_pow2k_denom(ptr addrspace(1) %out,
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: v_mov_b32_e32 v0, 0x80080081
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: v_mul_hi_i32 v0, s5, v0
-; GFX6-NEXT: s_ashr_i32 s6, s4, 31
-; GFX6-NEXT: s_lshr_b32 s6, s6, 20
-; GFX6-NEXT: s_add_i32 s4, s4, s6
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, s5, v0
-; GFX6-NEXT: s_ashr_i32 s4, s4, 12
+; GFX6-NEXT: v_mul_hi_i32 v0, s3, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_ashr_i32 s0, s2, 31
+; GFX6-NEXT: s_lshr_b32 s0, s0, 20
+; GFX6-NEXT: s_add_i32 s0, s2, s0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, s3, v0
+; GFX6-NEXT: s_ashr_i32 s0, s0, 12
; GFX6-NEXT: v_lshrrev_b32_e32 v1, 31, v0
; GFX6-NEXT: v_ashrrev_i32_e32 v0, 11, v0
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_add_i32_e32 v1, vcc, v0, v1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: ssdiv_v2i32_mixed_pow2k_denom:
@@ -6453,136 +6474,138 @@ define amdgpu_kernel void @sdiv_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
;
; GFX6-LABEL: sdiv_v2i32_pow2_shl_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_lshl_b32 s2, 0x1000, s2
-; GFX6-NEXT: s_abs_i32 s6, s2
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX6-NEXT: s_sub_i32 s7, 0, s6
-; GFX6-NEXT: s_lshl_b32 s3, 0x1000, s3
+; GFX6-NEXT: s_lshl_b32 s6, 0x1000, s6
+; GFX6-NEXT: s_abs_i32 s8, s6
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s8
+; GFX6-NEXT: s_sub_i32 s0, 0, s8
+; GFX6-NEXT: s_lshl_b32 s9, 0x1000, s7
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s7, v0
-; GFX6-NEXT: s_abs_i32 s7, s0
-; GFX6-NEXT: s_xor_b32 s0, s0, s2
-; GFX6-NEXT: s_ashr_i32 s0, s0, 31
+; GFX6-NEXT: v_mul_lo_u32 v1, s0, v0
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_abs_i32 s4, s2
+; GFX6-NEXT: s_xor_b32 s2, s2, s6
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s7, v0
-; GFX6-NEXT: v_readfirstlane_b32 s2, v0
-; GFX6-NEXT: s_mul_i32 s2, s2, s6
-; GFX6-NEXT: s_sub_i32 s2, s7, s2
-; GFX6-NEXT: s_sub_i32 s7, s2, s6
+; GFX6-NEXT: v_mul_hi_u32 v0, s4, v0
+; GFX6-NEXT: s_ashr_i32 s2, s2, 31
+; GFX6-NEXT: s_mov_b32 s6, -1
+; GFX6-NEXT: v_readfirstlane_b32 s5, v0
+; GFX6-NEXT: s_mul_i32 s5, s5, s8
+; GFX6-NEXT: s_sub_i32 s4, s4, s5
+; GFX6-NEXT: s_sub_i32 s5, s4, s8
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT: s_cmp_ge_u32 s2, s6
+; GFX6-NEXT: s_cmp_ge_u32 s4, s8
; GFX6-NEXT: s_cselect_b64 vcc, -1, 0
; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT: s_cselect_b32 s2, s7, s2
+; GFX6-NEXT: s_cselect_b32 s4, s5, s4
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; GFX6-NEXT: s_cmp_ge_u32 s2, s6
+; GFX6-NEXT: s_cmp_ge_u32 s4, s8
; GFX6-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX6-NEXT: s_abs_i32 s2, s3
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s2
-; GFX6-NEXT: s_sub_i32 s6, 0, s2
+; GFX6-NEXT: s_abs_i32 s8, s9
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s8
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_sub_i32 s0, 0, s8
; GFX6-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX6-NEXT: s_xor_b32 s3, s1, s3
; GFX6-NEXT: v_rcp_f32_e32 v2, v2
-; GFX6-NEXT: s_abs_i32 s1, s1
-; GFX6-NEXT: v_xor_b32_e32 v0, s0, v0
-; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s0, v0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_abs_i32 s1, s3
+; GFX6-NEXT: v_xor_b32_e32 v0, s2, v0
; GFX6-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
; GFX6-NEXT: v_cvt_u32_f32_e32 v2, v2
-; GFX6-NEXT: s_ashr_i32 s3, s3, 31
-; GFX6-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-NEXT: v_mul_lo_u32 v3, s6, v2
-; GFX6-NEXT: s_mov_b32 s6, -1
+; GFX6-NEXT: v_subrev_i32_e32 v0, vcc, s2, v0
+; GFX6-NEXT: v_mul_lo_u32 v3, s0, v2
+; GFX6-NEXT: s_xor_b32 s0, s3, s9
+; GFX6-NEXT: s_ashr_i32 s0, s0, 31
; GFX6-NEXT: v_mul_hi_u32 v1, v2, v3
; GFX6-NEXT: v_add_i32_e32 v1, vcc, v2, v1
; GFX6-NEXT: v_mul_hi_u32 v1, s1, v1
-; GFX6-NEXT: v_readfirstlane_b32 s0, v1
-; GFX6-NEXT: s_mul_i32 s0, s0, s2
-; GFX6-NEXT: s_sub_i32 s0, s1, s0
-; GFX6-NEXT: s_sub_i32 s1, s0, s2
+; GFX6-NEXT: v_readfirstlane_b32 s2, v1
+; GFX6-NEXT: s_mul_i32 s2, s2, s8
+; GFX6-NEXT: s_sub_i32 s1, s1, s2
+; GFX6-NEXT: s_sub_i32 s2, s1, s8
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v1
-; GFX6-NEXT: s_cmp_ge_u32 s0, s2
+; GFX6-NEXT: s_cmp_ge_u32 s1, s8
; GFX6-NEXT: s_cselect_b64 vcc, -1, 0
; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX6-NEXT: s_cselect_b32 s0, s1, s0
+; GFX6-NEXT: s_cselect_b32 s1, s2, s1
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 1, v1
-; GFX6-NEXT: s_cmp_ge_u32 s0, s2
+; GFX6-NEXT: s_cmp_ge_u32 s1, s8
; GFX6-NEXT: s_cselect_b64 vcc, -1, 0
; GFX6-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; GFX6-NEXT: v_xor_b32_e32 v1, s3, v1
-; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s3, v1
+; GFX6-NEXT: v_xor_b32_e32 v1, s0, v1
+; GFX6-NEXT: v_subrev_i32_e32 v1, vcc, s0, v1
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: sdiv_v2i32_pow2_shl_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s2, 0x1000, s2
-; GFX9-NEXT: s_abs_i32 s6, s2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX9-NEXT: s_lshl_b32 s7, 0x1000, s3
-; GFX9-NEXT: s_abs_i32 s3, s0
-; GFX9-NEXT: s_xor_b32 s0, s0, s2
+; GFX9-NEXT: s_lshl_b32 s6, 0x1000, s6
+; GFX9-NEXT: s_abs_i32 s8, s6
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s8
+; GFX9-NEXT: s_lshl_b32 s4, 0x1000, s7
+; GFX9-NEXT: s_abs_i32 s5, s2
+; GFX9-NEXT: s_xor_b32 s2, s2, s6
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: s_sub_i32 s2, 0, s6
-; GFX9-NEXT: s_ashr_i32 s0, s0, 31
+; GFX9-NEXT: s_sub_i32 s6, 0, s8
+; GFX9-NEXT: s_ashr_i32 s2, s2, 31
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s8, v0
-; GFX9-NEXT: s_mul_i32 s2, s2, s8
-; GFX9-NEXT: s_mul_hi_u32 s2, s8, s2
-; GFX9-NEXT: s_add_i32 s8, s8, s2
-; GFX9-NEXT: s_mul_hi_u32 s2, s3, s8
-; GFX9-NEXT: s_mul_i32 s8, s2, s6
-; GFX9-NEXT: s_sub_i32 s3, s3, s8
-; GFX9-NEXT: s_add_i32 s9, s2, 1
-; GFX9-NEXT: s_sub_i32 s8, s3, s6
-; GFX9-NEXT: s_cmp_ge_u32 s3, s6
-; GFX9-NEXT: s_cselect_b32 s2, s9, s2
-; GFX9-NEXT: s_cselect_b32 s3, s8, s3
-; GFX9-NEXT: s_add_i32 s8, s2, 1
-; GFX9-NEXT: s_cmp_ge_u32 s3, s6
-; GFX9-NEXT: s_cselect_b32 s6, s8, s2
-; GFX9-NEXT: s_abs_i32 s8, s7
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s8
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_xor_b32 s5, s6, s0
-; GFX9-NEXT: s_sub_i32 s6, 0, s8
+; GFX9-NEXT: v_readfirstlane_b32 s7, v0
+; GFX9-NEXT: s_mul_i32 s6, s6, s7
+; GFX9-NEXT: s_mul_hi_u32 s6, s7, s6
+; GFX9-NEXT: s_add_i32 s7, s7, s6
+; GFX9-NEXT: s_mul_hi_u32 s6, s5, s7
+; GFX9-NEXT: s_mul_i32 s7, s6, s8
+; GFX9-NEXT: s_sub_i32 s5, s5, s7
+; GFX9-NEXT: s_add_i32 s9, s6, 1
+; GFX9-NEXT: s_sub_i32 s7, s5, s8
+; GFX9-NEXT: s_cmp_ge_u32 s5, s8
+; GFX9-NEXT: s_cselect_b32 s6, s9, s6
+; GFX9-NEXT: s_cselect_b32 s5, s7, s5
+; GFX9-NEXT: s_add_i32 s7, s6, 1
+; GFX9-NEXT: s_cmp_ge_u32 s5, s8
+; GFX9-NEXT: s_cselect_b32 s5, s7, s6
+; GFX9-NEXT: s_abs_i32 s6, s4
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6
+; GFX9-NEXT: s_xor_b32 s5, s5, s2
+; GFX9-NEXT: s_sub_i32 s7, 0, s6
+; GFX9-NEXT: s_sub_i32 s2, s5, s2
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: s_sub_i32 s0, s5, s0
-; GFX9-NEXT: s_xor_b32 s4, s1, s7
-; GFX9-NEXT: s_abs_i32 s1, s1
+; GFX9-NEXT: s_xor_b32 s4, s3, s4
+; GFX9-NEXT: s_abs_i32 s3, s3
+; GFX9-NEXT: s_ashr_i32 s4, s4, 31
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: s_ashr_i32 s4, s4, 31
; GFX9-NEXT: v_readfirstlane_b32 s5, v0
-; GFX9-NEXT: s_mul_i32 s6, s6, s5
-; GFX9-NEXT: s_mul_hi_u32 s6, s5, s6
-; GFX9-NEXT: s_add_i32 s5, s5, s6
-; GFX9-NEXT: s_mul_hi_u32 s5, s1, s5
-; GFX9-NEXT: s_mul_i32 s6, s5, s8
-; GFX9-NEXT: s_sub_i32 s1, s1, s6
+; GFX9-NEXT: s_mul_i32 s7, s7, s5
+; GFX9-NEXT: s_mul_hi_u32 s7, s5, s7
+; GFX9-NEXT: s_add_i32 s5, s5, s7
+; GFX9-NEXT: s_mul_hi_u32 s5, s3, s5
+; GFX9-NEXT: s_mul_i32 s7, s5, s6
+; GFX9-NEXT: s_sub_i32 s3, s3, s7
+; GFX9-NEXT: s_add_i32 s8, s5, 1
+; GFX9-NEXT: s_sub_i32 s7, s3, s6
+; GFX9-NEXT: s_cmp_ge_u32 s3, s6
+; GFX9-NEXT: s_cselect_b32 s5, s8, s5
+; GFX9-NEXT: s_cselect_b32 s3, s7, s3
; GFX9-NEXT: s_add_i32 s7, s5, 1
-; GFX9-NEXT: s_sub_i32 s6, s1, s8
-; GFX9-NEXT: s_cmp_ge_u32 s1, s8
-; GFX9-NEXT: s_cselect_b32 s5, s7, s5
-; GFX9-NEXT: s_cselect_b32 s1, s6, s1
-; GFX9-NEXT: s_add_i32 s6, s5, 1
-; GFX9-NEXT: s_cmp_ge_u32 s1, s8
-; GFX9-NEXT: s_cselect_b32 s1, s6, s5
-; GFX9-NEXT: s_xor_b32 s1, s1, s4
-; GFX9-NEXT: s_sub_i32 s1, s1, s4
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT: s_cmp_ge_u32 s3, s6
+; GFX9-NEXT: s_cselect_b32 s3, s7, s5
+; GFX9-NEXT: s_xor_b32 s3, s3, s4
+; GFX9-NEXT: s_sub_i32 s3, s3, s4
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
%shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
%r = sdiv <2 x i32> %x, %shl.y
@@ -6598,28 +6621,28 @@ define amdgpu_kernel void @srem_i32_oddk_denom(ptr addrspace(1) %out, i32 %x) {
;
; GFX6-LABEL: srem_i32_oddk_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: v_mov_b32_e32 v0, 0xd9528441
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_mul_hi_i32 v0, s6, v0
-; GFX6-NEXT: v_readfirstlane_b32 s4, v0
-; GFX6-NEXT: s_add_i32 s4, s4, s6
-; GFX6-NEXT: s_lshr_b32 s5, s4, 31
-; GFX6-NEXT: s_ashr_i32 s4, s4, 20
-; GFX6-NEXT: s_add_i32 s4, s4, s5
-; GFX6-NEXT: s_mul_i32 s4, s4, 0x12d8fb
-; GFX6-NEXT: s_sub_i32 s4, s6, s4
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: v_mul_hi_i32 v0, s2, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_add_i32 s0, s0, s2
+; GFX6-NEXT: s_lshr_b32 s1, s0, 31
+; GFX6-NEXT: s_ashr_i32 s0, s0, 20
+; GFX6-NEXT: s_add_i32 s0, s0, s1
+; GFX6-NEXT: s_mul_i32 s0, s0, 0x12d8fb
+; GFX6-NEXT: s_sub_i32 s0, s2, s0
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: srem_i32_oddk_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_mul_hi_i32 s3, s2, 0xd9528441
@@ -6645,24 +6668,24 @@ define amdgpu_kernel void @srem_i32_pow2k_denom(ptr addrspace(1) %out, i32 %x) {
;
; GFX6-LABEL: srem_i32_pow2k_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_ashr_i32 s4, s6, 31
-; GFX6-NEXT: s_lshr_b32 s4, s4, 20
-; GFX6-NEXT: s_add_i32 s4, s6, s4
-; GFX6-NEXT: s_and_b32 s4, s4, 0xfffff000
-; GFX6-NEXT: s_sub_i32 s4, s6, s4
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_ashr_i32 s0, s2, 31
+; GFX6-NEXT: s_lshr_b32 s0, s0, 20
+; GFX6-NEXT: s_add_i32 s0, s2, s0
+; GFX6-NEXT: s_and_b32 s0, s0, 0xfffff000
+; GFX6-NEXT: s_sub_i32 s0, s2, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: srem_i32_pow2k_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_ashr_i32 s3, s2, 31
@@ -6688,36 +6711,37 @@ define amdgpu_kernel void @srem_i32_pow2_shl_denom(ptr addrspace(1) %out, i32 %x
; GFX6-LABEL: srem_i32_pow2_shl_denom:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_lshl_b32 s2, 0x1000, s5
-; GFX6-NEXT: s_abs_i32 s5, s2
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s5
-; GFX6-NEXT: s_sub_i32 s2, 0, s5
-; GFX6-NEXT: s_abs_i32 s6, s4
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_lshl_b32 s3, 0x1000, s3
+; GFX6-NEXT: s_abs_i32 s3, s3
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3
+; GFX6-NEXT: s_sub_i32 s4, 0, s3
+; GFX6-NEXT: s_abs_i32 s8, s2
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_rcp_iflag_f32_e32 v0, v0
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s2, v0
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: v_mul_lo_u32 v1, s4, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s6, v0
-; GFX6-NEXT: v_readfirstlane_b32 s7, v0
-; GFX6-NEXT: s_mul_i32 s7, s7, s5
-; GFX6-NEXT: s_sub_i32 s6, s6, s7
-; GFX6-NEXT: s_sub_i32 s7, s6, s5
-; GFX6-NEXT: s_cmp_ge_u32 s6, s5
-; GFX6-NEXT: s_cselect_b32 s6, s7, s6
-; GFX6-NEXT: s_sub_i32 s7, s6, s5
-; GFX6-NEXT: s_cmp_ge_u32 s6, s5
-; GFX6-NEXT: s_cselect_b32 s5, s7, s6
-; GFX6-NEXT: s_ashr_i32 s4, s4, 31
-; GFX6-NEXT: s_xor_b32 s5, s5, s4
-; GFX6-NEXT: s_sub_i32 s4, s5, s4
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: v_mul_hi_u32 v0, s8, v0
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_mul_i32 s0, s0, s3
+; GFX6-NEXT: s_sub_i32 s0, s8, s0
+; GFX6-NEXT: s_sub_i32 s1, s0, s3
+; GFX6-NEXT: s_cmp_ge_u32 s0, s3
+; GFX6-NEXT: s_cselect_b32 s0, s1, s0
+; GFX6-NEXT: s_sub_i32 s1, s0, s3
+; GFX6-NEXT: s_cmp_ge_u32 s0, s3
+; GFX6-NEXT: s_cselect_b32 s0, s1, s0
+; GFX6-NEXT: s_ashr_i32 s1, s2, 31
+; GFX6-NEXT: s_xor_b32 s0, s0, s1
+; GFX6-NEXT: s_sub_i32 s0, s0, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: srem_i32_pow2_shl_denom:
@@ -6772,23 +6796,24 @@ define amdgpu_kernel void @srem_v2i32_pow2k_denom(ptr addrspace(1) %out, <2 x i3
; GFX6-LABEL: srem_v2i32_pow2k_denom:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_ashr_i32 s6, s4, 31
-; GFX6-NEXT: s_lshr_b32 s6, s6, 20
-; GFX6-NEXT: s_ashr_i32 s7, s5, 31
-; GFX6-NEXT: s_add_i32 s6, s4, s6
-; GFX6-NEXT: s_lshr_b32 s7, s7, 20
-; GFX6-NEXT: s_and_b32 s6, s6, 0xfffff000
-; GFX6-NEXT: s_sub_i32 s4, s4, s6
-; GFX6-NEXT: s_add_i32 s6, s5, s7
-; GFX6-NEXT: s_and_b32 s6, s6, 0xfffff000
-; GFX6-NEXT: s_sub_i32 s5, s5, s6
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: v_mov_b32_e32 v1, s5
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_ashr_i32 s0, s2, 31
+; GFX6-NEXT: s_ashr_i32 s1, s3, 31
+; GFX6-NEXT: s_lshr_b32 s0, s0, 20
+; GFX6-NEXT: s_lshr_b32 s1, s1, 20
+; GFX6-NEXT: s_add_i32 s0, s2, s0
+; GFX6-NEXT: s_add_i32 s1, s3, s1
+; GFX6-NEXT: s_and_b32 s0, s0, 0xfffff000
+; GFX6-NEXT: s_and_b32 s1, s1, 0xfffff000
+; GFX6-NEXT: s_sub_i32 s0, s2, s0
+; GFX6-NEXT: s_sub_i32 s1, s3, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: srem_v2i32_pow2k_denom:
@@ -6899,122 +6924,125 @@ define amdgpu_kernel void @srem_v2i32_pow2_shl_denom(ptr addrspace(1) %out, <2 x
;
; GFX6-LABEL: srem_v2i32_pow2_shl_denom:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_lshl_b32 s2, 0x1000, s2
-; GFX6-NEXT: s_abs_i32 s2, s2
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX6-NEXT: s_sub_i32 s6, 0, s2
-; GFX6-NEXT: s_lshl_b32 s3, 0x1000, s3
+; GFX6-NEXT: s_lshl_b32 s0, 0x1000, s6
+; GFX6-NEXT: s_abs_i32 s6, s0
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s6
+; GFX6-NEXT: s_sub_i32 s0, 0, s6
; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s6, v0
-; GFX6-NEXT: s_abs_i32 s6, s0
-; GFX6-NEXT: s_ashr_i32 s0, s0, 31
+; GFX6-NEXT: v_mul_lo_u32 v1, s0, v0
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_lshl_b32 s5, 0x1000, s7
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_abs_i32 s4, s2
+; GFX6-NEXT: s_ashr_i32 s2, s2, 31
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s6, v0
+; GFX6-NEXT: v_mul_hi_u32 v0, s4, v0
; GFX6-NEXT: v_readfirstlane_b32 s7, v0
-; GFX6-NEXT: s_mul_i32 s7, s7, s2
-; GFX6-NEXT: s_sub_i32 s6, s6, s7
-; GFX6-NEXT: s_sub_i32 s7, s6, s2
-; GFX6-NEXT: s_cmp_ge_u32 s6, s2
-; GFX6-NEXT: s_cselect_b32 s6, s7, s6
-; GFX6-NEXT: s_sub_i32 s7, s6, s2
-; GFX6-NEXT: s_cmp_ge_u32 s6, s2
-; GFX6-NEXT: s_cselect_b32 s2, s7, s6
-; GFX6-NEXT: s_abs_i32 s3, s3
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s3
-; GFX6-NEXT: s_sub_i32 s6, 0, s3
-; GFX6-NEXT: s_abs_i32 s8, s1
-; GFX6-NEXT: s_xor_b32 s2, s2, s0
+; GFX6-NEXT: s_mul_i32 s7, s7, s6
+; GFX6-NEXT: s_sub_i32 s4, s4, s7
+; GFX6-NEXT: s_sub_i32 s7, s4, s6
+; GFX6-NEXT: s_cmp_ge_u32 s4, s6
+; GFX6-NEXT: s_cselect_b32 s4, s7, s4
+; GFX6-NEXT: s_sub_i32 s7, s4, s6
+; GFX6-NEXT: s_cmp_ge_u32 s4, s6
+; GFX6-NEXT: s_cselect_b32 s8, s7, s4
+; GFX6-NEXT: s_abs_i32 s9, s5
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s9
+; GFX6-NEXT: s_sub_i32 s4, 0, s9
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_ashr_i32 s1, s3, 31
; GFX6-NEXT: v_rcp_f32_e32 v0, v0
-; GFX6-NEXT: s_sub_i32 s0, s2, s0
-; GFX6-NEXT: s_ashr_i32 s1, s1, 31
; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s6, v0
-; GFX6-NEXT: s_mov_b32 s6, -1
+; GFX6-NEXT: v_mul_lo_u32 v1, s4, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_abs_i32 s0, s3
+; GFX6-NEXT: s_xor_b32 s3, s8, s2
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT: s_sub_i32 s2, s3, s2
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s8, v0
-; GFX6-NEXT: v_readfirstlane_b32 s2, v0
-; GFX6-NEXT: s_mul_i32 s2, s2, s3
-; GFX6-NEXT: s_sub_i32 s2, s8, s2
-; GFX6-NEXT: s_sub_i32 s8, s2, s3
-; GFX6-NEXT: s_cmp_ge_u32 s2, s3
-; GFX6-NEXT: s_cselect_b32 s2, s8, s2
-; GFX6-NEXT: s_sub_i32 s8, s2, s3
-; GFX6-NEXT: s_cmp_ge_u32 s2, s3
-; GFX6-NEXT: s_cselect_b32 s2, s8, s2
-; GFX6-NEXT: s_xor_b32 s2, s2, s1
-; GFX6-NEXT: s_sub_i32 s1, s2, s1
-; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX6-NEXT: v_readfirstlane_b32 s3, v0
+; GFX6-NEXT: s_mul_i32 s3, s3, s9
+; GFX6-NEXT: s_sub_i32 s0, s0, s3
+; GFX6-NEXT: s_sub_i32 s3, s0, s9
+; GFX6-NEXT: s_cmp_ge_u32 s0, s9
+; GFX6-NEXT: s_cselect_b32 s0, s3, s0
+; GFX6-NEXT: s_sub_i32 s3, s0, s9
+; GFX6-NEXT: s_cmp_ge_u32 s0, s9
+; GFX6-NEXT: s_cselect_b32 s0, s3, s0
+; GFX6-NEXT: s_xor_b32 s0, s0, s1
+; GFX6-NEXT: s_sub_i32 s0, s0, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: srem_v2i32_pow2_shl_denom:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s2, 0x1000, s2
-; GFX9-NEXT: s_abs_i32 s2, s2
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX9-NEXT: s_sub_i32 s7, 0, s2
-; GFX9-NEXT: s_ashr_i32 s6, s0, 31
-; GFX9-NEXT: s_abs_i32 s0, s0
+; GFX9-NEXT: s_lshl_b32 s0, 0x1000, s6
+; GFX9-NEXT: s_abs_i32 s6, s0
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_lshl_b32 s4, 0x1000, s7
+; GFX9-NEXT: s_sub_i32 s7, 0, s6
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: s_lshl_b32 s3, 0x1000, s3
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_ashr_i32 s5, s2, 31
+; GFX9-NEXT: s_abs_i32 s2, s2
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX9-NEXT: v_readfirstlane_b32 s8, v0
; GFX9-NEXT: s_mul_i32 s7, s7, s8
; GFX9-NEXT: s_mul_hi_u32 s7, s8, s7
; GFX9-NEXT: s_add_i32 s8, s8, s7
-; GFX9-NEXT: s_mul_hi_u32 s7, s0, s8
-; GFX9-NEXT: s_mul_i32 s7, s7, s2
-; GFX9-NEXT: s_sub_i32 s0, s0, s7
-; GFX9-NEXT: s_sub_i32 s7, s0, s2
-; GFX9-NEXT: s_cmp_ge_u32 s0, s2
-; GFX9-NEXT: s_cselect_b32 s0, s7, s0
-; GFX9-NEXT: s_sub_i32 s7, s0, s2
-; GFX9-NEXT: s_cmp_ge_u32 s0, s2
-; GFX9-NEXT: s_cselect_b32 s0, s7, s0
-; GFX9-NEXT: s_abs_i32 s7, s3
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s7
-; GFX9-NEXT: s_xor_b32 s0, s0, s6
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_sub_i32 s5, 0, s7
+; GFX9-NEXT: s_mul_hi_u32 s7, s2, s8
+; GFX9-NEXT: s_mul_i32 s7, s7, s6
+; GFX9-NEXT: s_sub_i32 s2, s2, s7
+; GFX9-NEXT: s_sub_i32 s7, s2, s6
+; GFX9-NEXT: s_cmp_ge_u32 s2, s6
+; GFX9-NEXT: s_cselect_b32 s2, s7, s2
+; GFX9-NEXT: s_sub_i32 s7, s2, s6
+; GFX9-NEXT: s_cmp_ge_u32 s2, s6
+; GFX9-NEXT: s_cselect_b32 s2, s7, s2
+; GFX9-NEXT: s_abs_i32 s4, s4
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s4
+; GFX9-NEXT: s_xor_b32 s2, s2, s5
+; GFX9-NEXT: s_sub_i32 s7, 0, s4
+; GFX9-NEXT: s_sub_i32 s2, s2, s5
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
-; GFX9-NEXT: s_sub_i32 s0, s0, s6
-; GFX9-NEXT: s_ashr_i32 s4, s1, 31
-; GFX9-NEXT: s_abs_i32 s1, s1
+; GFX9-NEXT: s_ashr_i32 s6, s3, 31
+; GFX9-NEXT: s_abs_i32 s3, s3
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s6, v0
-; GFX9-NEXT: s_mul_i32 s5, s5, s6
-; GFX9-NEXT: s_mul_hi_u32 s5, s6, s5
-; GFX9-NEXT: s_add_i32 s6, s6, s5
-; GFX9-NEXT: s_mul_hi_u32 s5, s1, s6
-; GFX9-NEXT: s_mul_i32 s5, s5, s7
-; GFX9-NEXT: s_sub_i32 s1, s1, s5
-; GFX9-NEXT: s_sub_i32 s5, s1, s7
-; GFX9-NEXT: s_cmp_ge_u32 s1, s7
-; GFX9-NEXT: s_cselect_b32 s1, s5, s1
-; GFX9-NEXT: s_sub_i32 s5, s1, s7
-; GFX9-NEXT: s_cmp_ge_u32 s1, s7
-; GFX9-NEXT: s_cselect_b32 s1, s5, s1
-; GFX9-NEXT: s_xor_b32 s1, s1, s4
-; GFX9-NEXT: s_sub_i32 s1, s1, s4
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT: v_readfirstlane_b32 s5, v0
+; GFX9-NEXT: s_mul_i32 s7, s7, s5
+; GFX9-NEXT: s_mul_hi_u32 s7, s5, s7
+; GFX9-NEXT: s_add_i32 s5, s5, s7
+; GFX9-NEXT: s_mul_hi_u32 s5, s3, s5
+; GFX9-NEXT: s_mul_i32 s5, s5, s4
+; GFX9-NEXT: s_sub_i32 s3, s3, s5
+; GFX9-NEXT: s_sub_i32 s5, s3, s4
+; GFX9-NEXT: s_cmp_ge_u32 s3, s4
+; GFX9-NEXT: s_cselect_b32 s3, s5, s3
+; GFX9-NEXT: s_sub_i32 s5, s3, s4
+; GFX9-NEXT: s_cmp_ge_u32 s3, s4
+; GFX9-NEXT: s_cselect_b32 s3, s5, s3
+; GFX9-NEXT: s_xor_b32 s3, s3, s6
+; GFX9-NEXT: s_sub_i32 s3, s3, s6
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
%shl.y = shl <2 x i32> <i32 4096, i32 4096>, %y
%r = srem <2 x i32> %x, %shl.y
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-miscellaneous-uniform-intrinsic.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-miscellaneous-uniform-intrinsic.ll
index 92198f2593f3a..9d47d847889d6 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-miscellaneous-uniform-intrinsic.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-miscellaneous-uniform-intrinsic.ll
@@ -79,9 +79,7 @@ define amdgpu_kernel void @readlane_readlane(ptr addrspace(1) %out) {
define amdgpu_kernel void @permlane64_uniform(ptr addrspace(1) %out, i32 %src) {
; CHECK-LABEL: permlane64_uniform:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_clause 0x1
-; CHECK-NEXT: s_load_b32 s2, s[4:5], 0x8
-; CHECK-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; CHECK-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; CHECK-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll b/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
index 85b5c7c870b23..4171a5b31c0e0 100644
--- a/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdhsa-trap-num-sgprs.ll
@@ -2,8 +2,8 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx803 -mattr=-trap-handler < %s | FileCheck %s --check-prefixes=GCN,TRAP-HANDLER-DISABLE
; GCN-LABEL: {{^}}amdhsa_trap_num_sgprs
-; TRAP-HANDLER-ENABLE: NumSgprs: 61
-; TRAP-HANDLER-DISABLE: NumSgprs: 77
+; TRAP-HANDLER-ENABLE: NumSgprs: 60
+; TRAP-HANDLER-DISABLE: NumSgprs: 60
define amdgpu_kernel void @amdhsa_trap_num_sgprs(
ptr addrspace(1) %out0, i32 %in0,
ptr addrspace(1) %out1, i32 %in1,
diff --git a/llvm/test/CodeGen/AMDGPU/and.ll b/llvm/test/CodeGen/AMDGPU/and.ll
index fe9ec8e6ef52a..77a41f63741a7 100644
--- a/llvm/test/CodeGen/AMDGPU/and.ll
+++ b/llvm/test/CodeGen/AMDGPU/and.ll
@@ -123,25 +123,27 @@ define amdgpu_kernel void @s_and_i32(ptr addrspace(1) %out, i32 %a, i32 %b) {
; GFX6-LABEL: s_and_i32:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_and_b32 s4, s4, s5
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_and_b32 s0, s2, s3
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: s_and_i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX8-NEXT: s_and_b32 s4, s4, s5
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_and_b32 s0, s2, s3
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
%and = and i32 %a, %b
store i32 %and, ptr addrspace(1) %out, align 4
@@ -151,26 +153,28 @@ define amdgpu_kernel void @s_and_i32(ptr addrspace(1) %out, i32 %a, i32 %b) {
define amdgpu_kernel void @s_and_constant_i32(ptr addrspace(1) %out, i32 %a) {
; GFX6-LABEL: s_and_constant_i32:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_and_b32 s4, s6, 0x12d687
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_and_b32 s0, s2, 0x12d687
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: s_and_constant_i32:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s4, s6, 0x12d687
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_and_b32 s0, s2, 0x12d687
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
%and = and i32 %a, 1234567
store i32 %and, ptr addrspace(1) %out, align 4
@@ -187,34 +191,36 @@ define amdgpu_kernel void @s_and_multi_use_constant_i32_0(ptr addrspace(1) %out,
; GFX6-LABEL: s_and_multi_use_constant_i32_0:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GFX6-NEXT: v_mov_b32_e32 v0, 0x12d687
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_and_b32 s4, s4, 0x12d687
-; GFX6-NEXT: s_add_i32 s4, s4, s5
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v1, s4
-; GFX6-NEXT: buffer_store_dword v1, off, s[0:3], 0
-; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_and_b32 s0, s2, 0x12d687
+; GFX6-NEXT: s_add_i32 s0, s0, s3
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0)
+; GFX6-NEXT: v_mov_b32_e32 v0, 0x12d687
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: s_and_multi_use_constant_i32_0:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX8-NEXT: v_mov_b32_e32 v0, 0x12d687
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX8-NEXT: s_and_b32 s4, s4, 0x12d687
-; GFX8-NEXT: s_add_i32 s4, s4, s5
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_mov_b32_e32 v1, s4
-; GFX8-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_and_b32 s0, s2, 0x12d687
+; GFX8-NEXT: s_add_i32 s0, s0, s3
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: v_mov_b32_e32 v0, 0x12d687
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_endpgm
%and = and i32 %a, 1234567
@@ -232,30 +238,32 @@ define amdgpu_kernel void @s_and_multi_use_constant_i32_1(ptr addrspace(1) %out,
; GFX6-LABEL: s_and_multi_use_constant_i32_1:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_and_b32 s4, s4, 0x12d687
-; GFX6-NEXT: s_add_i32 s4, s4, s5
-; GFX6-NEXT: s_add_i32 s4, s4, 0x12d687
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_and_b32 s0, s2, 0x12d687
+; GFX6-NEXT: s_add_i32 s0, s0, s3
+; GFX6-NEXT: s_add_i32 s0, s0, 0x12d687
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: s_and_multi_use_constant_i32_1:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX8-NEXT: s_and_b32 s4, s4, 0x12d687
-; GFX8-NEXT: s_add_i32 s4, s4, s5
-; GFX8-NEXT: s_add_i32 s4, s4, 0x12d687
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_and_b32 s0, s2, 0x12d687
+; GFX8-NEXT: s_add_i32 s0, s0, s3
+; GFX8-NEXT: s_add_i32 s0, s0, 0x12d687
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: s_endpgm
%and = and i32 %a, 1234567
@@ -327,9 +335,10 @@ define amdgpu_kernel void @v_and_i32_sgpr_vgpr(ptr addrspace(1) %out, i32 %a, pt
; GFX6-NEXT: v_mov_b32_e32 v1, 0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_mov_b64 s[0:1], s[4:5]
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_and_b32_e32 v2, s6, v2
; GFX6-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT: s_endpgm
@@ -343,8 +352,7 @@ define amdgpu_kernel void @v_and_i32_sgpr_vgpr(ptr addrspace(1) %out, i32 %a, pt
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
@@ -573,30 +581,32 @@ define amdgpu_kernel void @s_and_i64(ptr addrspace(1) %out, i64 %a, i64 %b) {
define amdgpu_kernel void @s_and_i1(ptr addrspace(1) %out, i1 %a, i1 %b) {
; GFX6-LABEL: s_and_i1:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_lshr_b32 s4, s6, 8
-; GFX6-NEXT: s_and_b32 s4, s6, s4
-; GFX6-NEXT: s_and_b32 s4, s4, 1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_lshr_b32 s0, s2, 8
+; GFX6-NEXT: s_and_b32 s0, s2, s0
+; GFX6-NEXT: s_and_b32 s0, s0, 1
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_byte v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: s_and_i1:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s4, s6, 8
-; GFX8-NEXT: s_and_b32 s4, s6, s4
-; GFX8-NEXT: s_and_b32 s4, s4, 1
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_lshr_b32 s0, s2, 8
+; GFX8-NEXT: s_and_b32 s0, s2, s0
+; GFX8-NEXT: s_and_b32 s0, s0, 1
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_byte v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
%and = and i1 %a, %b
store i1 %and, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/andorbitset.ll b/llvm/test/CodeGen/AMDGPU/andorbitset.ll
index 52321c8ab038e..5a21785fe82ac 100644
--- a/llvm/test/CodeGen/AMDGPU/andorbitset.ll
+++ b/llvm/test/CodeGen/AMDGPU/andorbitset.ll
@@ -4,14 +4,15 @@
define amdgpu_kernel void @s_clear_msb(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_clear_msb:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0x7fffffff
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0x7fffffff
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = and i32 %in, 2147483647
store i32 %x, ptr addrspace(1) %out
@@ -21,14 +22,15 @@ define amdgpu_kernel void @s_clear_msb(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @s_set_msb(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_set_msb:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_or_b32 s4, s6, 0x80000000
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_or_b32 s0, s2, 0x80000000
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = or i32 %in, 2147483648
store i32 %x, ptr addrspace(1) %out
@@ -38,14 +40,15 @@ define amdgpu_kernel void @s_set_msb(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @s_clear_lsb(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_clear_lsb:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, -2
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, -2
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = and i32 %in, 4294967294
store i32 %x, ptr addrspace(1) %out
@@ -55,14 +58,15 @@ define amdgpu_kernel void @s_clear_lsb(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @s_set_lsb(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_set_lsb:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_or_b32 s4, s6, 1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_or_b32 s0, s2, 1
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = or i32 %in, 1
store i32 %x, ptr addrspace(1) %out
@@ -72,14 +76,15 @@ define amdgpu_kernel void @s_set_lsb(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @s_clear_midbit(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_clear_midbit:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0xfffffeff
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0xfffffeff
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = and i32 %in, 4294967039
store i32 %x, ptr addrspace(1) %out
@@ -89,14 +94,15 @@ define amdgpu_kernel void @s_clear_midbit(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @s_set_midbit(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_set_midbit:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_or_b32 s4, s6, 0x100
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_or_b32 s0, s2, 0x100
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = or i32 %in, 256
store i32 %x, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/andorn2.ll b/llvm/test/CodeGen/AMDGPU/andorn2.ll
index c9ad669ad5b20..871c8fef3889a 100644
--- a/llvm/test/CodeGen/AMDGPU/andorn2.ll
+++ b/llvm/test/CodeGen/AMDGPU/andorn2.ll
@@ -9,35 +9,37 @@ define amdgpu_kernel void @scalar_andn2_i32_one_use(ptr addrspace(1) %r0, i32 %a
; GFX6-LABEL: scalar_andn2_i32_one_use:
; GFX6: ; %bb.0: ; %entry
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_andn2_b32 s4, s4, s5
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_andn2_b32 s0, s2, s3
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: scalar_andn2_i32_one_use:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX7-NEXT: s_andn2_b32 s4, s4, s5
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_andn2_b32 s0, s2, s3
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: scalar_andn2_i32_one_use:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_andn2_b32 s2, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_andn2_b32 s0, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -145,35 +147,37 @@ define amdgpu_kernel void @scalar_orn2_i32_one_use(ptr addrspace(1) %r0, i32 %a,
; GFX6-LABEL: scalar_orn2_i32_one_use:
; GFX6: ; %bb.0: ; %entry
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_orn2_b32 s4, s4, s5
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_orn2_b32 s0, s2, s3
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: scalar_orn2_i32_one_use:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX7-NEXT: s_orn2_b32 s4, s4, s5
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_mov_b32_e32 v0, s4
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_orn2_b32 s0, s2, s3
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_mov_b32_e32 v0, s0
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: scalar_orn2_i32_one_use:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_orn2_b32 s2, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_orn2_b32 s0, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -280,41 +284,41 @@ entry:
define amdgpu_kernel void @vector_andn2_i32_s_v_one_use(ptr addrspace(1) %r0, i32 %s) {
; GFX6-LABEL: vector_andn2_i32_s_v_one_use:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_bfi_b32 v0, v0, 0, s6
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_bfi_b32 v0, v0, 0, s2
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: vector_andn2_i32_s_v_one_use:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v0, 0, s6
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_bfi_b32 v0, v0, 0, s2
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: vector_andn2_i32_s_v_one_use:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v0, 0, s2
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_store_dword v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_bfi_b32 v0, v0, 0, s2
+; GFX8-NEXT: flat_store_dword v[1:2], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: vector_andn2_i32_s_v_one_use:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_bfi_b32 v0, v0, 0, s2
@@ -341,41 +345,41 @@ entry:
define amdgpu_kernel void @vector_andn2_i32_v_s_one_use(ptr addrspace(1) %r0, i32 %s) {
; GFX6-LABEL: vector_andn2_i32_v_s_one_use:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_bfi_b32 v0, s6, 0, v0
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_bfi_b32 v0, s2, 0, v0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: vector_andn2_i32_v_s_one_use:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, s6, 0, v0
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_bfi_b32 v0, s2, 0, v0
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: vector_andn2_i32_v_s_one_use:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, s2, 0, v0
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_store_dword v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_bfi_b32 v0, s2, 0, v0
+; GFX8-NEXT: flat_store_dword v[1:2], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: vector_andn2_i32_v_s_one_use:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_bfi_b32 v0, s2, 0, v0
@@ -402,41 +406,41 @@ entry:
define amdgpu_kernel void @vector_orn2_i32_s_v_one_use(ptr addrspace(1) %r0, i32 %s) {
; GFX6-LABEL: vector_orn2_i32_s_v_one_use:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_bfi_b32 v0, v0, s6, -1
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_bfi_b32 v0, v0, s2, -1
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: vector_orn2_i32_s_v_one_use:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, v0, s6, -1
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_bfi_b32 v0, v0, s2, -1
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: vector_orn2_i32_s_v_one_use:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, v0, s2, -1
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_store_dword v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_bfi_b32 v0, v0, s2, -1
+; GFX8-NEXT: flat_store_dword v[1:2], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: vector_orn2_i32_s_v_one_use:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_bfi_b32 v0, v0, s2, -1
@@ -464,41 +468,41 @@ entry:
define amdgpu_kernel void @vector_orn2_i32_v_s_one_use(ptr addrspace(1) %r0, i32 %s) {
; GFX6-LABEL: vector_orn2_i32_v_s_one_use:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_bfi_b32 v0, s6, v0, -1
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_bfi_b32 v0, s2, v0, -1
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: vector_orn2_i32_v_s_one_use:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_bfi_b32 v0, s6, v0, -1
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_bfi_b32 v0, s2, v0, -1
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: vector_orn2_i32_v_s_one_use:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_bfi_b32 v2, s2, v0, -1
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_store_dword v[0:1], v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_bfi_b32 v0, s2, v0, -1
+; GFX8-NEXT: flat_store_dword v[1:2], v0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: vector_orn2_i32_v_s_one_use:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_bfi_b32 v0, s2, v0, -1
@@ -802,35 +806,37 @@ define amdgpu_kernel void @vector_andn2_v2i32_s_v_one_use(ptr addrspace(1) %r0,
; GFX6-LABEL: vector_andn2_v2i32_s_v_one_use:
; GFX6: ; %bb.0: ; %entry
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_bfi_b32 v1, v1, 0, s5
-; GFX6-NEXT: v_bfi_b32 v0, v0, 0, s4
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_bfi_b32 v1, v1, 0, s3
+; GFX6-NEXT: v_bfi_b32 v0, v0, 0, s2
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: vector_andn2_v2i32_s_v_one_use:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_bfi_b32 v1, v1, 0, s5
-; GFX7-NEXT: v_bfi_b32 v0, v0, 0, s4
-; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_bfi_b32 v1, v1, 0, s3
+; GFX7-NEXT: v_bfi_b32 v0, v0, 0, s2
+; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: vector_andn2_v2i32_s_v_one_use:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_bfi_b32 v1, v1, 0, s3
; GFX8-NEXT: v_bfi_b32 v0, v0, 0, s2
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
@@ -871,35 +877,37 @@ define amdgpu_kernel void @vector_andn2_v2i32_v_s_one_use(ptr addrspace(1) %r0,
; GFX6-LABEL: vector_andn2_v2i32_v_s_one_use:
; GFX6: ; %bb.0: ; %entry
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_bfi_b32 v1, s5, 0, v1
-; GFX6-NEXT: v_bfi_b32 v0, s4, 0, v0
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_bfi_b32 v1, s3, 0, v1
+; GFX6-NEXT: v_bfi_b32 v0, s2, 0, v0
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: vector_andn2_v2i32_v_s_one_use:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_bfi_b32 v1, s5, 0, v1
-; GFX7-NEXT: v_bfi_b32 v0, s4, 0, v0
-; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_bfi_b32 v1, s3, 0, v1
+; GFX7-NEXT: v_bfi_b32 v0, s2, 0, v0
+; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: vector_andn2_v2i32_v_s_one_use:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_bfi_b32 v1, s3, 0, v1
; GFX8-NEXT: v_bfi_b32 v0, s2, 0, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
@@ -939,12 +947,14 @@ entry:
define amdgpu_kernel void @vector_andn2_v2i32_s_s_one_use(ptr addrspace(1) %r0, <2 x i32> %s1, <2 x i32> %s2) {
; GFX6-LABEL: vector_andn2_v2i32_s_s_one_use:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_andn2_b64 s[0:1], s[2:3], s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, s0
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -952,12 +962,14 @@ define amdgpu_kernel void @vector_andn2_v2i32_s_s_one_use(ptr addrspace(1) %r0,
;
; GFX7-LABEL: vector_andn2_v2i32_s_s_one_use:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: s_andn2_b64 s[0:1], s[2:3], s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -965,39 +977,39 @@ define amdgpu_kernel void @vector_andn2_v2i32_s_s_one_use(ptr addrspace(1) %r0,
;
; GFX8-LABEL: vector_andn2_v2i32_s_s_one_use:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: s_andn2_b64 s[0:1], s[2:3], s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: vector_andn2_v2i32_s_s_one_use:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: s_andn2_b64 s[2:3], s[2:3], s[6:7]
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX12-LABEL: vector_andn2_v2i32_s_s_one_use:
; GFX12: ; %bb.0: ; %entry
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT: s_and_not1_b64 s[2:3], s[2:3], s[4:5]
; GFX12-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
entry:
%not = xor <2 x i32> %s2, <i32 -1, i32 -1>
@@ -1239,35 +1251,37 @@ define amdgpu_kernel void @vector_orn2_v2i32_s_v_one_use(ptr addrspace(1) %r0, <
; GFX6-LABEL: vector_orn2_v2i32_s_v_one_use:
; GFX6: ; %bb.0: ; %entry
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_bfi_b32 v1, v1, s5, -1
-; GFX6-NEXT: v_bfi_b32 v0, v0, s4, -1
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_bfi_b32 v1, v1, s3, -1
+; GFX6-NEXT: v_bfi_b32 v0, v0, s2, -1
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: vector_orn2_v2i32_s_v_one_use:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_bfi_b32 v1, v1, s5, -1
-; GFX7-NEXT: v_bfi_b32 v0, v0, s4, -1
-; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_bfi_b32 v1, v1, s3, -1
+; GFX7-NEXT: v_bfi_b32 v0, v0, s2, -1
+; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: vector_orn2_v2i32_s_v_one_use:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_bfi_b32 v1, v1, s3, -1
; GFX8-NEXT: v_bfi_b32 v0, v0, s2, -1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
@@ -1308,35 +1322,37 @@ define amdgpu_kernel void @vector_orn2_v2i32_v_s_one_use(ptr addrspace(1) %r0, <
; GFX6-LABEL: vector_orn2_v2i32_v_s_one_use:
; GFX6: ; %bb.0: ; %entry
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_bfi_b32 v1, s5, v1, -1
-; GFX6-NEXT: v_bfi_b32 v0, s4, v0, -1
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_bfi_b32 v1, s3, v1, -1
+; GFX6-NEXT: v_bfi_b32 v0, s2, v0, -1
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX7-LABEL: vector_orn2_v2i32_v_s_one_use:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: v_bfi_b32 v1, s5, v1, -1
-; GFX7-NEXT: v_bfi_b32 v0, s4, v0, -1
-; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_bfi_b32 v1, s3, v1, -1
+; GFX7-NEXT: v_bfi_b32 v0, s2, v0, -1
+; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: vector_orn2_v2i32_v_s_one_use:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_bfi_b32 v1, s3, v1, -1
; GFX8-NEXT: v_bfi_b32 v0, s2, v0, -1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
@@ -1376,12 +1392,14 @@ entry:
define amdgpu_kernel void @vector_orn2_v2i32_s_s_one_use(ptr addrspace(1) %r0, <2 x i32> %s1, <2 x i32> %s2) {
; GFX6-LABEL: vector_orn2_v2i32_s_s_one_use:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_orn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_orn2_b64 s[0:1], s[2:3], s[8:9]
; GFX6-NEXT: v_mov_b32_e32 v0, s0
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -1389,12 +1407,14 @@ define amdgpu_kernel void @vector_orn2_v2i32_s_s_one_use(ptr addrspace(1) %r0, <
;
; GFX7-LABEL: vector_orn2_v2i32_s_s_one_use:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_orn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: s_orn2_b64 s[0:1], s[2:3], s[8:9]
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
@@ -1402,39 +1422,39 @@ define amdgpu_kernel void @vector_orn2_v2i32_s_s_one_use(ptr addrspace(1) %r0, <
;
; GFX8-LABEL: vector_orn2_v2i32_s_s_one_use:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_orn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: s_orn2_b64 s[0:1], s[2:3], s[6:7]
; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v1, s7
; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: vector_orn2_v2i32_s_s_one_use:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_orn2_b64 s[0:1], s[0:1], s[2:3]
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: s_orn2_b64 s[2:3], s[2:3], s[6:7]
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX12-LABEL: vector_orn2_v2i32_s_s_one_use:
; GFX12: ; %bb.0: ; %entry
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_or_not1_b64 s[0:1], s[0:1], s[2:3]
+; GFX12-NEXT: s_or_not1_b64 s[2:3], s[2:3], s[4:5]
; GFX12-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
entry:
%not = xor <2 x i32> %s2, <i32 -1, i32 -1>
diff --git a/llvm/test/CodeGen/AMDGPU/andorxorinvimm.ll b/llvm/test/CodeGen/AMDGPU/andorxorinvimm.ll
index 41951587de0e0..1f4e6edb27777 100644
--- a/llvm/test/CodeGen/AMDGPU/andorxorinvimm.ll
+++ b/llvm/test/CodeGen/AMDGPU/andorxorinvimm.ll
@@ -4,14 +4,15 @@
define amdgpu_kernel void @s_or_to_orn2(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_or_to_orn2:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_or_b32 s4, s6, 0xffffffcd
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_or_b32 s0, s2, 0xffffffcd
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = or i32 %in, -51
store i32 %x, ptr addrspace(1) %out
@@ -21,14 +22,15 @@ define amdgpu_kernel void @s_or_to_orn2(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @s_or_to_orn2_imm0(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_or_to_orn2_imm0:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_or_b32 s4, s6, 0xffffffcd
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_or_b32 s0, s2, 0xffffffcd
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = or i32 -51, %in
store i32 %x, ptr addrspace(1) %out
@@ -38,14 +40,15 @@ define amdgpu_kernel void @s_or_to_orn2_imm0(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @s_and_to_andn2(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_and_to_andn2:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0xffffffcd
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0xffffffcd
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = and i32 %in, -51
store i32 %x, ptr addrspace(1) %out
@@ -55,14 +58,15 @@ define amdgpu_kernel void @s_and_to_andn2(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @s_and_to_andn2_imm0(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_and_to_andn2_imm0:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0xffffffcd
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0xffffffcd
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = and i32 -51, %in
store i32 %x, ptr addrspace(1) %out
@@ -72,14 +76,15 @@ define amdgpu_kernel void @s_and_to_andn2_imm0(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @s_xor_to_xnor(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_xor_to_xnor:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_xor_b32 s4, s6, 0xffffffcd
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_xor_b32 s0, s2, 0xffffffcd
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = xor i32 %in, -51
store i32 %x, ptr addrspace(1) %out
@@ -89,14 +94,15 @@ define amdgpu_kernel void @s_xor_to_xnor(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @s_xor_to_xnor_imm0(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_xor_to_xnor_imm0:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_xor_b32 s4, s6, 0xffffffcd
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_xor_b32 s0, s2, 0xffffffcd
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
%x = xor i32 -51, %in
store i32 %x, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/anyext.ll b/llvm/test/CodeGen/AMDGPU/anyext.ll
index 1492119a6022d..b2c947456145d 100644
--- a/llvm/test/CodeGen/AMDGPU/anyext.ll
+++ b/llvm/test/CodeGen/AMDGPU/anyext.ll
@@ -9,41 +9,44 @@ declare i32 @llvm.amdgcn.workitem.id.y() nounwind readnone
define amdgpu_kernel void @anyext_i1_i32(ptr addrspace(1) %out, i32 %cond) #0 {
; GCN-LABEL: anyext_i1_i32:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_lg_u32 s6, 0
-; GCN-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_cmp_lg_u32 s2, 0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
;
; GFX8-LABEL: anyext_i1_i32:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_cmp_lg_u32 s6, 0
-; GFX8-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_cmp_lg_u32 s2, 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: anyext_i1_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_cmp_lg_u32 s6, 0
-; GFX9-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: s_cmp_lg_u32 s2, 0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
entry:
%tmp = icmp eq i32 %cond, 0
diff --git a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
index 2a27d7ff34bd5..8a501d2521c12 100644
--- a/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/asyncmark-gfx12plus.ll
@@ -396,9 +396,9 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
; GISEL-NEXT: v_mov_b32_e32 v0, 0
; GISEL-NEXT: s_load_b32 s11, s[4:5], 0x44 nv
-; GISEL-NEXT: s_mov_b32 s13, 2
+; GISEL-NEXT: s_mov_b32 s12, 2
; GISEL-NEXT: s_wait_kmcnt 0x0
-; GISEL-NEXT: s_load_b32 s12, s[8:9], 0x0
+; GISEL-NEXT: s_load_b32 s13, s[8:9], 0x0
; GISEL-NEXT: s_load_b32 s17, s[0:1], 0x0
; GISEL-NEXT: v_mov_b32_e32 v1, s10
; GISEL-NEXT: s_add_co_u32 s6, s10, 4
@@ -433,15 +433,15 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: v_mov_b32_e32 v3, s8
; GISEL-NEXT: s_mov_b32 s18, s9
; GISEL-NEXT: s_wait_kmcnt 0x0
-; GISEL-NEXT: s_add_co_i32 s9, s12, s17
-; GISEL-NEXT: s_add_co_i32 s13, s13, 1
+; GISEL-NEXT: s_add_co_i32 s9, s13, s17
+; GISEL-NEXT: s_add_co_i32 s12, s12, 1
; GISEL-NEXT: s_mov_b32 s8, s19
; GISEL-NEXT: global_load_async_to_lds_b32 v3, v0, s[6:7] offset:4 nv
; GISEL-NEXT: v_mov_b32_e32 v3, s14
; GISEL-NEXT: ; asyncmark
; GISEL-NEXT: ; wait_asyncmark(2)
; GISEL-NEXT: s_wait_asynccnt 0x2
-; GISEL-NEXT: s_mov_b32 s12, s15
+; GISEL-NEXT: s_mov_b32 s13, s15
; GISEL-NEXT: ds_load_b32 v3, v3
; GISEL-NEXT: s_wait_dscnt 0x0
; GISEL-NEXT: v_readfirstlane_b32 s17, v3
@@ -458,7 +458,7 @@ define amdgpu_kernel void @test_pipelined_loop_with_global(ptr addrspace(1) %foo
; GISEL-NEXT: s_add_co_u32 s4, s4, 4
; GISEL-NEXT: s_add_co_ci_u32 s5, s5, 0
; GISEL-NEXT: s_add_co_u32 s14, s14, 4
-; GISEL-NEXT: s_cmp_lt_i32 s13, s11
+; GISEL-NEXT: s_cmp_lt_i32 s12, s11
; GISEL-NEXT: s_wait_loadcnt 0x1
; GISEL-NEXT: v_readfirstlane_b32 s9, v1
; GISEL-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll b/llvm/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll
index 49977a4c64784..297643f7c1892 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_cmp_swap_local.ll
@@ -118,16 +118,15 @@ define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i64_offset(ptr addrspace(1) %o
;
; GFX7-LABEL: lds_atomic_cmpxchg_ret_i64_offset:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0xd
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; GFX7-NEXT: v_mov_b32_e32 v0, 7
; GFX7-NEXT: v_mov_b32_e32 v1, 0
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v4, s6
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-NEXT: v_mov_b32_e32 v3, s3
; GFX7-NEXT: s_mov_b32 m0, -1
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_mov_b32_e32 v4, s2
+; GFX7-NEXT: v_mov_b32_e32 v2, s4
+; GFX7-NEXT: v_mov_b32_e32 v3, s5
; GFX7-NEXT: ds_cmpst_rtn_b64 v[0:1], v4, v[0:1], v[2:3] offset:32
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 s3, 0xf000
@@ -137,53 +136,49 @@ define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i64_offset(ptr addrspace(1) %o
;
; GFX8-LABEL: lds_atomic_cmpxchg_ret_i64_offset:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; GFX8-NEXT: v_mov_b32_e32 v0, 7
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v4, s6
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: s_mov_b32 m0, -1
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s4
+; GFX8-NEXT: v_mov_b32_e32 v3, s5
; GFX8-NEXT: ds_cmpst_rtn_b64 v[0:1], v4, v[0:1], v[2:3] offset:32
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s3
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_cmpxchg_ret_i64_offset:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 7
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v4, s6
-; GFX9-NEXT: v_mov_b32_e32 v3, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s0
+; GFX9-NEXT: v_mov_b32_e32 v4, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_mov_b32_e32 v3, s7
; GFX9-NEXT: ds_cmpst_rtn_b64 v[0:1], v4, v[0:1], v[2:3] offset:32
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: lds_atomic_cmpxchg_ret_i64_offset:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX11-NEXT: v_mov_b32_e32 v0, 7
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v4, s2
-; GFX11-NEXT: v_mov_b32_e32 v3, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v2, s4
+; GFX11-NEXT: v_mov_b32_e32 v3, s5
; GFX11-NEXT: ds_cmpstore_rtn_b64 v[0:1], v4, v[2:3], v[0:1] offset:32
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -222,70 +217,70 @@ define amdgpu_kernel void @lds_atomic_cmpxchg_ret_i32_bad_si_offset(ptr addrspac
;
; GFX7-LABEL: lds_atomic_cmpxchg_ret_i32_bad_si_offset:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX7-NEXT: v_mov_b32_e32 v0, 7
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_sub_i32 s2, s2, s3
-; GFX7-NEXT: v_mov_b32_e32 v1, s1
-; GFX7-NEXT: s_lshl_b32 s1, s2, 2
-; GFX7-NEXT: s_add_i32 s0, s0, s1
-; GFX7-NEXT: v_mov_b32_e32 v2, s0
+; GFX7-NEXT: s_sub_i32 s4, s6, s7
+; GFX7-NEXT: v_mov_b32_e32 v1, s3
+; GFX7-NEXT: s_lshl_b32 s3, s4, 2
+; GFX7-NEXT: s_add_i32 s2, s2, s3
+; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: ds_cmpst_rtn_b32 v0, v2, v0, v1 offset:16
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b32 s6, -1
-; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX7-NEXT: s_mov_b32 s3, 0xf000
+; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: lds_atomic_cmpxchg_ret_i32_bad_si_offset:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: v_mov_b32_e32 v0, 7
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sub_i32 s2, s2, s3
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: s_lshl_b32 s1, s2, 2
-; GFX8-NEXT: s_add_i32 s0, s0, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
+; GFX8-NEXT: s_sub_i32 s4, s6, s7
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: s_lshl_b32 s3, s4, 2
+; GFX8-NEXT: s_add_i32 s2, s2, s3
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
; GFX8-NEXT: ds_cmpst_rtn_b32 v2, v2, v0, v1 offset:16
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_cmpxchg_ret_i32_bad_si_offset:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 7
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sub_i32 s2, s2, s3
-; GFX9-NEXT: s_lshl2_add_u32 s0, s2, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: s_sub_i32 s4, s6, s7
+; GFX9-NEXT: s_lshl2_add_u32 s2, s4, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: ds_cmpst_rtn_b32 v0, v1, v0, v2 offset:16
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: global_store_dword v1, v0, s[6:7]
+; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: lds_atomic_cmpxchg_ret_i32_bad_si_offset:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 7
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_sub_i32 s2, s2, s3
-; GFX11-NEXT: v_mov_b32_e32 v2, s1
-; GFX11-NEXT: s_lshl2_add_u32 s0, s2, s0
+; GFX11-NEXT: s_sub_i32 s4, s6, s7
+; GFX11-NEXT: v_mov_b32_e32 v2, s3
+; GFX11-NEXT: s_lshl2_add_u32 s2, s4, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_mov_b32_e32 v1, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
; GFX11-NEXT: ds_cmpstore_rtn_b32 v0, v1, v2, v0 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
index aed1e4fe685f5..38cdf6a71c628 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_global_pointer.ll
@@ -4011,7 +4011,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX7LESS-NEXT: s_cbranch_execnz .LBB7_2
; GFX7LESS-NEXT: ; %bb.3: ; %Flow
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX7LESS-NEXT: .LBB7_4: ; %Flow3
+; GFX7LESS-NEXT: .LBB7_4: ; %Flow4
; GFX7LESS-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
@@ -4061,7 +4061,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX8-NEXT: s_cbranch_execnz .LBB7_2
; GFX8-NEXT: ; %bb.3: ; %Flow
; GFX8-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX8-NEXT: .LBB7_4: ; %Flow3
+; GFX8-NEXT: .LBB7_4: ; %Flow4
; GFX8-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mul_lo_u32 v1, s12, v2
@@ -4110,7 +4110,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX9-NEXT: s_cbranch_execnz .LBB7_2
; GFX9-NEXT: ; %bb.3: ; %Flow
; GFX9-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX9-NEXT: .LBB7_4: ; %Flow3
+; GFX9-NEXT: .LBB7_4: ; %Flow4
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mul_lo_u32 v1, s12, v2
@@ -4161,7 +4161,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1064-NEXT: s_cbranch_execnz .LBB7_2
; GFX1064-NEXT: ; %bb.3: ; %Flow
; GFX1064-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1064-NEXT: .LBB7_4: ; %Flow3
+; GFX1064-NEXT: .LBB7_4: ; %Flow4
; GFX1064-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mul_lo_u32 v1, s12, v2
@@ -4211,7 +4211,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1032-NEXT: s_cbranch_execnz .LBB7_2
; GFX1032-NEXT: ; %bb.3: ; %Flow
; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s10
-; GFX1032-NEXT: .LBB7_4: ; %Flow3
+; GFX1032-NEXT: .LBB7_4: ; %Flow4
; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mul_lo_u32 v1, s8, v2
@@ -4266,7 +4266,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1164-NEXT: s_cbranch_execnz .LBB7_2
; GFX1164-NEXT: ; %bb.3: ; %Flow
; GFX1164-NEXT: s_or_b64 exec, exec, s[10:11]
-; GFX1164-NEXT: .LBB7_4: ; %Flow3
+; GFX1164-NEXT: .LBB7_4: ; %Flow4
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
@@ -4320,7 +4320,7 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, ptr addrspace(
; GFX1132-NEXT: s_cbranch_execnz .LBB7_2
; GFX1132-NEXT: ; %bb.3: ; %Flow
; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s10
-; GFX1132-NEXT: .LBB7_4: ; %Flow3
+; GFX1132-NEXT: .LBB7_4: ; %Flow4
; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s9
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
index 29190ad5a695a..86932bb01edc2 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_optimizations_local_pointer.ll
@@ -228,122 +228,124 @@ entry:
define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive) {
; GFX7LESS-LABEL: add_i32_uniform:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB1_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s2, s6, s2
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX7LESS-NEXT: s_mul_i32 s3, s2, s3
; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
; GFX7LESS-NEXT: s_mov_b32 m0, -1
; GFX7LESS-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: .LBB1_2:
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s6, v0
-; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s4, v0
-; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_mov_b32 s4, s0
+; GFX7LESS-NEXT: s_mov_b32 s5, s1
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v1
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s2, v0
+; GFX7LESS-NEXT: v_add_i32_e32 v0, vcc, s0, v0
+; GFX7LESS-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7LESS-NEXT: s_endpgm
;
; GFX8-LABEL: add_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
-; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB1_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB1_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
-; GFX8-NEXT: v_readfirstlane_b32 s4, v1
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, s4, v0
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: v_readfirstlane_b32 s0, v1
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: add_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[6:7], exec
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB1_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB1_2:
-; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
-; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_add_u32_e32 v0, s4, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v1
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: v_add_u32_e32 v0, s0, v0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
;
; GFX1064-LABEL: add_i32_uniform:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_mov_b64 s[6:7], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB1_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s2, s6, s2
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
+; GFX1064-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: s_mul_i32 s3, s2, s3
+; GFX1064-NEXT: v_mov_b32_e32 v2, s3
; GFX1064-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
; GFX1064-NEXT: .LBB1_2:
; GFX1064-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1064-NEXT: v_readfirstlane_b32 s4, v1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s6, v0, s[2:3]
+; GFX1064-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v0, s[4:5]
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: s_mov_b32 s2, -1
; GFX1064-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -351,95 +353,89 @@ define amdgpu_kernel void @add_i32_uniform(ptr addrspace(1) %out, i32 %additive)
;
; GFX1032-LABEL: add_i32_uniform:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032-NEXT: s_mov_b32 s6, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB1_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s6
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mul_i32 s2, s0, s2
-; GFX1032-NEXT: v_mov_b32_e32 v2, s2
+; GFX1032-NEXT: s_mul_i32 s4, s2, s4
+; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: buffer_gl0_inv
; GFX1032-NEXT: .LBB1_2:
; GFX1032-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX1032-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
-; GFX1032-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1032-NEXT: s_mov_b32 s10, -1
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mad_u64_u32 v[0:1], s0, s0, v0, s[2:3]
-; GFX1032-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT: v_mad_u64_u32 v[0:1], s2, s2, v0, s[4:5]
+; GFX1032-NEXT: s_mov_b32 s2, -1
+; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: add_i32_uniform:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB1_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s2, s6, s2
+; GFX1164-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: s_mul_i32 s3, s2, s3
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164-NEXT: v_mov_b32_e32 v2, s3
; GFX1164-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
; GFX1164-NEXT: .LBB1_2:
-; GFX1164-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX1164-NEXT: v_readfirstlane_b32 s4, v1
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1164-NEXT: v_mad_u64_u32 v[1:2], null, s6, v0, s[2:3]
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1164-NEXT: v_mad_u64_u32 v[1:2], null, s2, v0, s[4:5]
; GFX1164-NEXT: s_mov_b32 s2, -1
; GFX1164-NEXT: buffer_store_b32 v1, off, s[0:3], 0
; GFX1164-NEXT: s_endpgm
;
; GFX1132-LABEL: add_i32_uniform:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_load_b32 s0, s[4:5], 0x2c
-; GFX1132-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB1_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mul_i32 s2, s0, s2
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s2
+; GFX1132-NEXT: s_bcnt1_i32_b32 s4, s6
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_mul_i32 s4, s2, s4
+; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s4
; GFX1132-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
; GFX1132-NEXT: .LBB1_2:
-; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX1132-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX1132-NEXT: v_readfirstlane_b32 s2, v1
-; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mad_u64_u32 v[1:2], null, s0, v0, s[2:3]
-; GFX1132-NEXT: buffer_store_b32 v1, off, s[4:7], 0
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1132-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT: v_mad_u64_u32 v[1:2], null, s2, v0, s[4:5]
+; GFX1132-NEXT: s_mov_b32 s2, -1
+; GFX1132-NEXT: buffer_store_b32 v1, off, s[0:3], 0
; GFX1132-NEXT: s_endpgm
entry:
%old = atomicrmw add ptr addrspace(3) @local_var32, i32 %additive acq_rel
@@ -3618,121 +3614,123 @@ entry:
define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive) {
; GFX7LESS-LABEL: sub_i32_uniform:
; GFX7LESS: ; %bb.0: ; %entry
-; GFX7LESS-NEXT: s_mov_b64 s[2:3], exec
-; GFX7LESS-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s2, 0
-; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s3, v0
+; GFX7LESS-NEXT: s_mov_b64 s[6:7], exec
+; GFX7LESS-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7LESS-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s6, 0
+; GFX7LESS-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s7, v0
; GFX7LESS-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7LESS-NEXT: ; implicit-def: $vgpr1
-; GFX7LESS-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX7LESS-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX7LESS-NEXT: s_cbranch_execz .LBB9_2
; GFX7LESS-NEXT: ; %bb.1:
-; GFX7LESS-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: s_mul_i32 s2, s6, s2
+; GFX7LESS-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX7LESS-NEXT: s_mul_i32 s3, s2, s3
; GFX7LESS-NEXT: v_mov_b32_e32 v1, 0
-; GFX7LESS-NEXT: v_mov_b32_e32 v2, s2
+; GFX7LESS-NEXT: v_mov_b32_e32 v2, s3
; GFX7LESS-NEXT: s_mov_b32 m0, -1
; GFX7LESS-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
; GFX7LESS-NEXT: .LBB9_2:
-; GFX7LESS-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX7LESS-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7LESS-NEXT: s_mov_b32 s3, 0xf000
-; GFX7LESS-NEXT: s_mov_b32 s2, -1
-; GFX7LESS-NEXT: v_readfirstlane_b32 s4, v1
+; GFX7LESS-NEXT: s_or_b64 exec, exec, s[4:5]
+; GFX7LESS-NEXT: s_mov_b32 s7, 0xf000
+; GFX7LESS-NEXT: s_mov_b32 s6, -1
; GFX7LESS-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7LESS-NEXT: v_mul_lo_u32 v0, s6, v0
-; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s4, v0
-; GFX7LESS-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7LESS-NEXT: s_mov_b32 s4, s0
+; GFX7LESS-NEXT: s_mov_b32 s5, s1
+; GFX7LESS-NEXT: v_readfirstlane_b32 s0, v1
+; GFX7LESS-NEXT: v_mul_lo_u32 v0, s2, v0
+; GFX7LESS-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
+; GFX7LESS-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7LESS-NEXT: s_endpgm
;
; GFX8-LABEL: sub_i32_uniform:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_mov_b64 s[2:3], exec
-; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b64 s[6:7], exec
+; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: ; implicit-def: $vgpr1
-; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX8-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX8-NEXT: s_cbranch_execz .LBB9_2
; GFX8-NEXT: ; %bb.1:
-; GFX8-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s2, s6, s2
+; GFX8-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX8-NEXT: s_mul_i32 s3, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v1, 0
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s3
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB9_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mul_lo_u32 v0, s6, v0
-; GFX8-NEXT: v_readfirstlane_b32 s4, v1
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s4, v0
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: v_mul_lo_u32 v0, s2, v0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: v_readfirstlane_b32 s0, v1
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_sub_u32_e32 v0, vcc, s0, v0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: sub_i32_uniform:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[6:7], exec
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB9_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_i32 s2, s6, s2
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX9-NEXT: s_mul_i32 s3, s2, s3
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB9_2:
-; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mul_lo_u32 v0, s6, v0
-; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_sub_u32_e32 v0, s4, v0
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: v_mul_lo_u32 v0, s2, v0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: v_readfirstlane_b32 s0, v1
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: v_sub_u32_e32 v0, s0, v0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
;
; GFX1064-LABEL: sub_i32_uniform:
; GFX1064: ; %bb.0: ; %entry
-; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_mov_b64 s[6:7], exec
; GFX1064-NEXT: ; implicit-def: $vgpr1
-; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1064-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX1064-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
; GFX1064-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; GFX1064-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX1064-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX1064-NEXT: s_cbranch_execz .LBB9_2
; GFX1064-NEXT: ; %bb.1:
-; GFX1064-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: s_mul_i32 s2, s6, s2
-; GFX1064-NEXT: v_mov_b32_e32 v2, s2
+; GFX1064-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX1064-NEXT: v_mov_b32_e32 v1, 0
+; GFX1064-NEXT: s_mul_i32 s3, s2, s3
+; GFX1064-NEXT: v_mov_b32_e32 v2, s3
; GFX1064-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: buffer_gl0_inv
; GFX1064-NEXT: .LBB9_2:
; GFX1064-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1064-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX1064-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX1064-NEXT: v_readfirstlane_b32 s2, v1
; GFX1064-NEXT: s_mov_b32 s3, 0x31016000
; GFX1064-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -3742,61 +3740,58 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX1032-LABEL: sub_i32_uniform:
; GFX1032: ; %bb.0: ; %entry
-; GFX1032-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX1032-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032-NEXT: s_mov_b32 s6, exec_lo
; GFX1032-NEXT: ; implicit-def: $vgpr1
-; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1032-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1032-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX1032-NEXT: s_and_saveexec_b32 s1, vcc_lo
+; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX1032-NEXT: s_cbranch_execz .LBB9_2
; GFX1032-NEXT: ; %bb.1:
-; GFX1032-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX1032-NEXT: s_bcnt1_i32_b32 s4, s6
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: s_mul_i32 s2, s0, s2
-; GFX1032-NEXT: v_mov_b32_e32 v2, s2
+; GFX1032-NEXT: s_mul_i32 s4, s2, s4
+; GFX1032-NEXT: v_mov_b32_e32 v2, s4
; GFX1032-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: buffer_gl0_inv
; GFX1032-NEXT: .LBB9_2:
; GFX1032-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX1032-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
-; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mul_lo_u32 v0, s0, v0
-; GFX1032-NEXT: v_readfirstlane_b32 s0, v1
-; GFX1032-NEXT: s_mov_b32 s11, 0x31016000
-; GFX1032-NEXT: s_mov_b32 s10, -1
-; GFX1032-NEXT: v_sub_nc_u32_e32 v0, s0, v0
-; GFX1032-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; GFX1032-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1032-NEXT: v_mul_lo_u32 v0, s2, v0
+; GFX1032-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1032-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1032-NEXT: v_sub_nc_u32_e32 v0, s2, v0
+; GFX1032-NEXT: s_mov_b32 s2, -1
+; GFX1032-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX1032-NEXT: s_endpgm
;
; GFX1164-LABEL: sub_i32_uniform:
; GFX1164: ; %bb.0: ; %entry
-; GFX1164-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1164-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164-NEXT: s_mov_b64 s[6:7], exec
+; GFX1164-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1164-NEXT: ; implicit-def: $vgpr1
; GFX1164-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX1164-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
; GFX1164-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1164-NEXT: s_cbranch_execz .LBB9_2
; GFX1164-NEXT: ; %bb.1:
-; GFX1164-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164-NEXT: v_mov_b32_e32 v1, 0
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: s_mul_i32 s2, s6, s2
+; GFX1164-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX1164-NEXT: v_mov_b32_e32 v1, 0
+; GFX1164-NEXT: s_mul_i32 s3, s2, s3
; GFX1164-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164-NEXT: v_mov_b32_e32 v2, s2
+; GFX1164-NEXT: v_mov_b32_e32 v2, s3
; GFX1164-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164-NEXT: buffer_gl0_inv
; GFX1164-NEXT: .LBB9_2:
-; GFX1164-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1164-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164-NEXT: v_mul_lo_u32 v0, s6, v0
+; GFX1164-NEXT: v_mul_lo_u32 v0, s2, v0
; GFX1164-NEXT: v_readfirstlane_b32 s2, v1
; GFX1164-NEXT: s_mov_b32 s3, 0x31016000
; GFX1164-NEXT: v_sub_nc_u32_e32 v0, s2, v0
@@ -3806,33 +3801,31 @@ define amdgpu_kernel void @sub_i32_uniform(ptr addrspace(1) %out, i32 %subitive)
;
; GFX1132-LABEL: sub_i32_uniform:
; GFX1132: ; %bb.0: ; %entry
-; GFX1132-NEXT: s_load_b32 s0, s[4:5], 0x2c
-; GFX1132-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132-NEXT: s_mov_b32 s1, exec_lo
-; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
+; GFX1132-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132-NEXT: s_mov_b32 s6, exec_lo
+; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX1132-NEXT: ; implicit-def: $vgpr1
; GFX1132-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX1132-NEXT: s_cbranch_execz .LBB9_2
; GFX1132-NEXT: ; %bb.1:
-; GFX1132-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: s_mul_i32 s2, s0, s2
-; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s2
+; GFX1132-NEXT: s_bcnt1_i32_b32 s4, s6
+; GFX1132-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132-NEXT: s_mul_i32 s4, s2, s4
+; GFX1132-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v2, s4
; GFX1132-NEXT: ds_sub_rtn_u32 v1, v1, v2
; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132-NEXT: buffer_gl0_inv
; GFX1132-NEXT: .LBB9_2:
-; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX1132-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX1132-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132-NEXT: v_mul_lo_u32 v0, s0, v0
-; GFX1132-NEXT: v_readfirstlane_b32 s0, v1
-; GFX1132-NEXT: s_mov_b32 s7, 0x31016000
-; GFX1132-NEXT: s_mov_b32 s6, -1
-; GFX1132-NEXT: v_sub_nc_u32_e32 v0, s0, v0
-; GFX1132-NEXT: buffer_store_b32 v0, off, s[4:7], 0
+; GFX1132-NEXT: s_or_b32 exec_lo, exec_lo, s3
+; GFX1132-NEXT: v_mul_lo_u32 v0, s2, v0
+; GFX1132-NEXT: v_readfirstlane_b32 s2, v1
+; GFX1132-NEXT: s_mov_b32 s3, 0x31016000
+; GFX1132-NEXT: v_sub_nc_u32_e32 v0, s2, v0
+; GFX1132-NEXT: s_mov_b32 s2, -1
+; GFX1132-NEXT: buffer_store_b32 v0, off, s[0:3], 0
; GFX1132-NEXT: s_endpgm
entry:
%old = atomicrmw sub ptr addrspace(3) @local_var32, i32 %subitive acq_rel
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll
index 0df8b3f8bdde3..3f42c721ba7f6 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_cond.ll
@@ -7,8 +7,7 @@
define amdgpu_kernel void @flat_atomic_usub_cond_no_rtn_u32(ptr %addr, i32 %in) {
; GFX9-SDAG-LABEL: flat_atomic_usub_cond_no_rtn_u32:
; GFX9-SDAG: ; %bb.0: ; %entry
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: s_add_u32 s0, s0, -16
; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, -1
@@ -99,23 +98,22 @@ entry:
define amdgpu_kernel void @flat_atomic_usub_cond_rtn_u32(ptr %addr, i32 %in, ptr %use) {
; GFX9-SDAG-LABEL: flat_atomic_usub_cond_rtn_u32:
; GFX9-SDAG: ; %bb.0: ; %entry
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_add_u32 s2, s0, 16
-; GFX9-SDAG-NEXT: s_addc_u32 s3, s1, 0
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: s_add_u32 s6, s0, 16
+; GFX9-SDAG-NEXT: s_addc_u32 s7, s1, 0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s7
; GFX9-SDAG-NEXT: flat_load_dword v0, v[0:1]
-; GFX9-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], 0
; GFX9-SDAG-NEXT: .LBB1_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
; GFX9-SDAG-NEXT: s_add_u32 s8, s0, 16
-; GFX9-SDAG-NEXT: v_subrev_u32_e32 v0, s6, v1
+; GFX9-SDAG-NEXT: v_subrev_u32_e32 v0, s2, v1
; GFX9-SDAG-NEXT: s_addc_u32 s9, s1, 0
-; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s6, v1
+; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s2, v1
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s8
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, s9
@@ -123,11 +121,11 @@ define amdgpu_kernel void @flat_atomic_usub_cond_rtn_u32(ptr %addr, i32 %in, ptr
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB1_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, s1
@@ -220,28 +218,27 @@ entry:
define amdgpu_kernel void @global_atomic_usub_cond_no_rtn_u32(ptr addrspace(1) %addr, i32 %in) {
; GFX9-SDAG-LABEL: global_atomic_usub_cond_no_rtn_u32:
; GFX9-SDAG: ; %bb.0: ; %entry
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_add_u32 s2, s0, -16
-; GFX9-SDAG-NEXT: s_addc_u32 s3, s1, -1
-; GFX9-SDAG-NEXT: s_load_dword s4, s[2:3], 0x0
-; GFX9-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-SDAG-NEXT: s_add_u32 s4, s0, -16
+; GFX9-SDAG-NEXT: s_addc_u32 s5, s1, -1
+; GFX9-SDAG-NEXT: s_load_dword s3, s[4:5], 0x0
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
; GFX9-SDAG-NEXT: .LBB2_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_subrev_u32_e32 v0, s6, v1
-; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s6, v1
+; GFX9-SDAG-NEXT: v_subrev_u32_e32 v0, s2, v1
+; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s2, v1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
; GFX9-SDAG-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:-16 glc
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB2_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-SDAG-NEXT: s_endpgm
@@ -306,29 +303,28 @@ entry:
define amdgpu_kernel void @global_atomic_usub_cond_rtn_u32(ptr addrspace(1) %addr, i32 %in, ptr addrspace(1) %use) {
; GFX9-SDAG-LABEL: global_atomic_usub_cond_rtn_u32:
; GFX9-SDAG: ; %bb.0: ; %entry
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-SDAG-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], 0
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_load_dword s7, s[0:1], 0x10
+; GFX9-SDAG-NEXT: s_load_dword s3, s[0:1], 0x10
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
; GFX9-SDAG-NEXT: .LBB3_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX9-SDAG-NEXT: v_subrev_u32_e32 v1, s6, v2
-; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s6, v2
+; GFX9-SDAG-NEXT: v_subrev_u32_e32 v1, s2, v2
+; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s2, v2
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
; GFX9-SDAG-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] offset:16 glc
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB3_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -337,11 +333,11 @@ define amdgpu_kernel void @global_atomic_usub_cond_rtn_u32(ptr addrspace(1) %add
;
; GFX12-SDAG-LABEL: global_atomic_usub_cond_rtn_u32:
; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX12-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-SDAG-NEXT: global_wb scope:SCOPE_SYS
; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
; GFX12-SDAG-NEXT: global_atomic_cond_sub_u32 v1, v0, v1, s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
@@ -893,31 +889,30 @@ define void @global_atomic_usub_cond_offset_nortn(ptr addrspace(1) %ptr, i32 %da
define amdgpu_kernel void @global_atomic_usub_cond_sgpr_base_offset(ptr addrspace(1) %ptr, i32 %data, ptr addrspace(1) %dst) {
; GFX9-SDAG-LABEL: global_atomic_usub_cond_sgpr_base_offset:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_mov_b64 s[6:7], 0
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_load_dword s7, s[2:3], 0x1000
-; GFX9-SDAG-NEXT: s_add_u32 s2, s2, 0x1000
-; GFX9-SDAG-NEXT: s_addc_u32 s3, s3, 0
+; GFX9-SDAG-NEXT: s_load_dword s3, s[0:1], 0x1000
+; GFX9-SDAG-NEXT: s_add_u32 s0, s0, 0x1000
+; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
; GFX9-SDAG-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, v1
-; GFX9-SDAG-NEXT: v_subrev_u32_e32 v1, s6, v2
-; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s6, v2
+; GFX9-SDAG-NEXT: v_subrev_u32_e32 v1, s2, v2
+; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s2, v2
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc
-; GFX9-SDAG-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[2:3] glc
+; GFX9-SDAG-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX9-SDAG-NEXT: s_or_b64 s[6:7], vcc, s[6:7]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB10_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -926,11 +921,11 @@ define amdgpu_kernel void @global_atomic_usub_cond_sgpr_base_offset(ptr addrspac
;
; GFX12-SDAG-LABEL: global_atomic_usub_cond_sgpr_base_offset:
; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX12-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-SDAG-NEXT: global_atomic_cond_sub_u32 v1, v0, v1, s[0:1] offset:4096 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
@@ -989,28 +984,27 @@ define amdgpu_kernel void @global_atomic_usub_cond_sgpr_base_offset(ptr addrspac
define amdgpu_kernel void @global_atomic_usub_cond_sgpr_base_offset_nortn(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-SDAG-LABEL: global_atomic_usub_cond_sgpr_base_offset_nortn:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_load_dword s4, s[2:3], 0x1000
-; GFX9-SDAG-NEXT: s_add_u32 s2, s2, 0x1000
-; GFX9-SDAG-NEXT: s_addc_u32 s3, s3, 0
+; GFX9-SDAG-NEXT: s_load_dword s3, s[0:1], 0x1000
+; GFX9-SDAG-NEXT: s_add_u32 s0, s0, 0x1000
+; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
; GFX9-SDAG-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_subrev_u32_e32 v0, s6, v1
-; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s6, v1
+; GFX9-SDAG-NEXT: v_subrev_u32_e32 v0, s2, v1
+; GFX9-SDAG-NEXT: v_cmp_le_u32_e32 vcc, s2, v1
; GFX9-SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GFX9-SDAG-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[2:3] glc
+; GFX9-SDAG-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB11_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-SDAG-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
index a49537c51adae..f7f0f14ca8870 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw_usub_sat.ll
@@ -497,27 +497,26 @@ define void @global_atomic_usub_sat_offset_nortn(ptr addrspace(1) %ptr, i32 %dat
define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset(ptr addrspace(1) %ptr, i32 %data, ptr addrspace(1) %dst) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x1000
+; GFX9-GISEL-NEXT: s_load_dword s3, s[0:1], 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX9-GISEL-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, v1
-; GFX9-GISEL-NEXT: v_sub_u32_e64 v1, v2, s4 clamp
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v1, v2, s2 clamp
; GFX9-GISEL-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v1, v2
-; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB4_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -526,35 +525,33 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset(ptr addrspace
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x2
-; GFX10-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x10
+; GFX10-GISEL-NEXT: s_clause 0x1
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-GISEL-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x1000
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX10-GISEL-NEXT: global_atomic_csub v0, v1, v0, s[0:1] glc
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: buffer_gl1_inv
; GFX10-GISEL-NEXT: buffer_gl0_inv
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-GISEL-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_clause 0x2
-; GFX11-GISEL-NEXT: s_load_b32 s6, s[4:5], 0x8
-; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-GISEL-NEXT: s_load_b64 s[2:3], s[4:5], 0x10
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x10
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s6
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s2
; GFX11-GISEL-NEXT: global_atomic_csub_u32 v0, v1, v0, s[0:1] glc
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: buffer_gl1_inv
; GFX11-GISEL-NEXT: buffer_gl0_inv
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-GISEL-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX11-GISEL-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset:
@@ -572,29 +569,28 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset(ptr addrspace
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0
-; GFX9-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_load_dword s5, s[2:3], 0x1000
-; GFX9-SDAG-NEXT: s_add_u32 s2, s2, 0x1000
-; GFX9-SDAG-NEXT: s_addc_u32 s3, s3, 0
+; GFX9-SDAG-NEXT: s_load_dword s3, s[0:1], 0x1000
+; GFX9-SDAG-NEXT: s_add_u32 s0, s0, 0x1000
+; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s5
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
; GFX9-SDAG-NEXT: .LBB4_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-SDAG-NEXT: v_sub_u32_e64 v2, v3, s4 clamp
-; GFX9-SDAG-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[2:3] glc
+; GFX9-SDAG-NEXT: v_sub_u32_e64 v2, v3, s2 clamp
+; GFX9-SDAG-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[0:1] glc
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
-; GFX9-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB4_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -603,44 +599,42 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset(ptr addrspace
;
; GFX10-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x2
-; GFX10-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x10
+; GFX10-SDAG-NEXT: s_clause 0x1
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-SDAG-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0x1000
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
; GFX10-SDAG-NEXT: global_atomic_csub v0, v0, v1, s[0:1] glc
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX10-SDAG-NEXT: buffer_gl1_inv
; GFX10-SDAG-NEXT: buffer_gl0_inv
; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[2:3]
+; GFX10-SDAG-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x2
-; GFX11-SDAG-NEXT: s_load_b32 s6, s[4:5], 0x8
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-SDAG-NEXT: s_load_b64 s[2:3], s[4:5], 0x10
+; GFX11-SDAG-NEXT: s_clause 0x1
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x10
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0x1000 :: v_dual_mov_b32 v1, s6
+; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0x1000 :: v_dual_mov_b32 v1, s2
; GFX11-SDAG-NEXT: global_atomic_csub_u32 v0, v0, v1, s[0:1] glc
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: buffer_gl1_inv
; GFX11-SDAG-NEXT: buffer_gl0_inv
; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-SDAG-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-SDAG-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX11-SDAG-NEXT: s_endpgm
;
; GFX12-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset:
; GFX12-SDAG: ; %bb.0:
+; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX12-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x10
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-SDAG-NEXT: global_atomic_sub_clamp_u32 v1, v0, v1, s[0:1] offset:4096 th:TH_ATOMIC_RETURN scope:SCOPE_DEV
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-NEXT: global_inv scope:SCOPE_DEV
@@ -655,33 +649,30 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset(ptr addrspace
define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn(ptr addrspace(1) %ptr, i32 %data) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x1000
+; GFX9-GISEL-NEXT: s_load_dword s3, s[0:1], 0x1000
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX9-GISEL-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_sub_u32_e64 v0, v1, s4 clamp
+; GFX9-GISEL-NEXT: v_sub_u32_e64 v0, v1, s2 clamp
; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB5_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0x1000
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, s2
@@ -693,9 +684,7 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn(ptr add
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_clause 0x1
-; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: v_dual_mov_b32 v1, 0x1000 :: v_dual_mov_b32 v0, s2
; GFX11-GISEL-NEXT: global_atomic_csub_u32 v0, v1, v0, s[0:1] glc
@@ -716,35 +705,32 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn(ptr add
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0
-; GFX9-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-SDAG-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_load_dword s5, s[2:3], 0x1000
-; GFX9-SDAG-NEXT: s_add_u32 s2, s2, 0x1000
-; GFX9-SDAG-NEXT: s_addc_u32 s3, s3, 0
+; GFX9-SDAG-NEXT: s_load_dword s3, s[0:1], 0x1000
+; GFX9-SDAG-NEXT: s_add_u32 s0, s0, 0x1000
+; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
; GFX9-SDAG-NEXT: .LBB5_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_sub_u32_e64 v0, v1, s4 clamp
-; GFX9-SDAG-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[2:3] glc
+; GFX9-SDAG-NEXT: v_sub_u32_e64 v0, v1, s2 clamp
+; GFX9-SDAG-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-SDAG-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB5_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX10-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0x1000
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
@@ -756,9 +742,7 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn(ptr add
;
; GFX11-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0x1000 :: v_dual_mov_b32 v1, s2
; GFX11-SDAG-NEXT: global_atomic_csub_u32 v0, v0, v1, s[0:1] glc
@@ -1672,29 +1656,28 @@ define void @global_atomic_usub_sat_offset_nortn_16(ptr addrspace(1) %ptr, i16 %
define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrspace(1) %ptr, i16 %data, ptr addrspace(1) %dst) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_16:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xffff0000
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x800
+; GFX9-GISEL-NEXT: s_load_dword s3, s[0:1], 0x800
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, s3
; GFX9-GISEL-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-GISEL-NEXT: v_sub_u16_e64 v2, v3, s4 clamp
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v2, v3, s2 clamp
; GFX9-GISEL-NEXT: v_and_or_b32 v2, v3, v0, v2
; GFX9-GISEL-NEXT: global_atomic_cmpswap v2, v1, v[2:3], s[0:1] offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB10_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -1703,9 +1686,7 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_16:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0x800
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: s_load_dword s3, s[0:1], 0x800
@@ -1736,9 +1717,7 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_16:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_clause 0x1
-; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x800
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -1804,29 +1783,28 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_16:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-SDAG-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-SDAG-NEXT: s_mov_b32 s5, 0xffff0000
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_mov_b32 s3, 0xffff0000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX9-SDAG-NEXT: s_load_dword s6, s[0:1], 0x800
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s6
; GFX9-SDAG-NEXT: .LBB10_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-SDAG-NEXT: v_sub_u16_e64 v0, v3, s4 clamp
-; GFX9-SDAG-NEXT: v_and_or_b32 v2, v3, s5, v0
+; GFX9-SDAG-NEXT: v_sub_u16_e64 v0, v3, s2 clamp
+; GFX9-SDAG-NEXT: v_and_or_b32 v2, v3, s3, v0
; GFX9-SDAG-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[0:1] offset:2048 glc
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
-; GFX9-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB10_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -1835,9 +1813,7 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
;
; GFX10-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_16:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-SDAG-NEXT: s_load_dword s3, s[0:1], 0x800
@@ -1870,9 +1846,7 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
;
; GFX11-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_16:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_load_b32 s3, s[0:1], 0x800
@@ -1946,35 +1920,32 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_16(ptr addrsp
define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr addrspace(1) %ptr, i16 %data) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_16:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xffff0000
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x800
+; GFX9-GISEL-NEXT: s_load_dword s3, s[0:1], 0x800
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX9-GISEL-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v1, s4 clamp
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v1, s2 clamp
; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, v2, v0
; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v3, v[0:1], s[0:1] offset:2048 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB11_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_16:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0x800
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-GISEL-NEXT: s_load_dword s3, s[0:1], 0x800
@@ -2000,9 +1971,7 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_16:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_clause 0x1
-; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x800
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -2056,35 +2025,32 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_16:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-SDAG-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-SDAG-NEXT: s_mov_b32 s5, 0xffff0000
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_mov_b32 s3, 0xffff0000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX9-SDAG-NEXT: s_load_dword s6, s[0:1], 0x800
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s6
; GFX9-SDAG-NEXT: .LBB11_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_sub_u16_e64 v0, v1, s4 clamp
-; GFX9-SDAG-NEXT: v_and_or_b32 v0, v1, s5, v0
+; GFX9-SDAG-NEXT: v_sub_u16_e64 v0, v1, s2 clamp
+; GFX9-SDAG-NEXT: v_and_or_b32 v0, v1, s3, v0
; GFX9-SDAG-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:2048 glc
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB11_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX10-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_16:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-SDAG-NEXT: s_load_dword s3, s[0:1], 0x800
@@ -2112,9 +2078,7 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_16(ptr
;
; GFX11-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_16:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: s_load_b32 s3, s[0:1], 0x800
@@ -3154,33 +3118,32 @@ define void @global_atomic_usub_sat_offset_nortn_8(ptr addrspace(1) %ptr, i8 %da
define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspace(1) %ptr, i8 %data, ptr addrspace(1) %dst) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0xff
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0xffffff00
-; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x400
-; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 8
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s5
+; GFX9-GISEL-NEXT: s_load_dword s3, s[0:1], 0x400
+; GFX9-GISEL-NEXT: s_lshl_b32 s2, s2, 8
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, s3
; GFX9-GISEL-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v3
; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v3, 8, v4
-; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v3, s4 clamp
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v3, v3, s2 clamp
; GFX9-GISEL-NEXT: v_and_b32_sdwa v3, v3, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_and_or_b32 v3, v4, v1, v3
; GFX9-GISEL-NEXT: global_atomic_cmpswap v3, v2, v[3:4], s[0:1] offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v3, v4
-; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB16_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -3189,22 +3152,20 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, 0xff
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: s_load_dword s2, s[0:1], 0x400
-; GFX10-GISEL-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-GISEL-NEXT: s_load_dword s3, s[0:1], 0x400
+; GFX10-GISEL-NEXT: s_lshl_b32 s2, s2, 8
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-GISEL-NEXT: s_mov_b32 s3, 0
; GFX10-GISEL-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, v2
; GFX10-GISEL-NEXT: v_lshlrev_b16 v2, 8, v3
-; GFX10-GISEL-NEXT: v_sub_nc_u16 v2, v2, s3 clamp
+; GFX10-GISEL-NEXT: v_sub_nc_u16 v2, v2, s2 clamp
; GFX10-GISEL-NEXT: v_and_b32_sdwa v2, v2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-GISEL-NEXT: v_and_or_b32 v2, 0xffffff00, v3, v2
; GFX10-GISEL-NEXT: global_atomic_cmpswap v2, v0, v[2:3], s[0:1] offset:1024 glc
@@ -3212,11 +3173,11 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX10-GISEL-NEXT: buffer_gl1_inv
; GFX10-GISEL-NEXT: buffer_gl0_inv
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v2, v3
-; GFX10-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX10-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB16_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX10-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -3225,16 +3186,14 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_clause 0x1
-; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-GISEL-NEXT: s_load_b32 s3, s[4:5], 0x8
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_load_b32 s2, s[0:1], 0x400
+; GFX11-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x400
; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-GISEL-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s2, 8
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: s_mov_b32 s3, 0
; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -3242,7 +3201,7 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, v1
; GFX11-GISEL-NEXT: v_lshlrev_b16 v1.l, 8, v2.l
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-GISEL-NEXT: v_sub_nc_u16 v1.l, v1.l, s3 clamp
+; GFX11-GISEL-NEXT: v_sub_nc_u16 v1.l, v1.l, s2 clamp
; GFX11-GISEL-NEXT: v_lshrrev_b16 v1.l, 8, v1.l
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xff, v1
@@ -3252,12 +3211,12 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX11-GISEL-NEXT: buffer_gl1_inv
; GFX11-GISEL-NEXT: buffer_gl0_inv
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB16_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x10
; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
@@ -3304,30 +3263,29 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-SDAG-NEXT: s_load_dword s5, s[8:9], 0x8
-; GFX9-SDAG-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-SDAG-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s3, 0xff00
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX9-SDAG-NEXT: s_load_dword s6, s[0:1], 0x400
-; GFX9-SDAG-NEXT: s_and_b32 s5, s5, 0xff
+; GFX9-SDAG-NEXT: s_and_b32 s2, s2, 0xff
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s6
; GFX9-SDAG-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-SDAG-NEXT: v_mov_b32_e32 v3, v0
-; GFX9-SDAG-NEXT: v_sub_u16_sdwa v0, v3, s5 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-SDAG-NEXT: v_and_or_b32 v2, v3, s4, v0
+; GFX9-SDAG-NEXT: v_sub_u16_sdwa v0, v3, s2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_and_or_b32 v2, v3, s3, v0
; GFX9-SDAG-NEXT: global_atomic_cmpswap v0, v1, v[2:3], s[0:1] offset:1024 glc
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v3
-; GFX9-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB16_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -3336,21 +3294,19 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
;
; GFX10-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-SDAG-NEXT: s_load_dword s3, s[8:9], 0x8
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: s_load_dword s2, s[0:1], 0x400
-; GFX10-SDAG-NEXT: s_and_b32 s3, s3, 0xff
+; GFX10-SDAG-NEXT: s_load_dword s3, s[0:1], 0x400
+; GFX10-SDAG-NEXT: s_and_b32 s2, s2, 0xff
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-SDAG-NEXT: s_mov_b32 s3, 0
; GFX10-SDAG-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX10-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, v1
; GFX10-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v2
-; GFX10-SDAG-NEXT: v_sub_nc_u16 v1, v1, s3 clamp
+; GFX10-SDAG-NEXT: v_sub_nc_u16 v1, v1, s2 clamp
; GFX10-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX10-SDAG-NEXT: v_and_or_b32 v1, 0xffffff00, v2, v1
; GFX10-SDAG-NEXT: global_atomic_cmpswap v1, v0, v[1:2], s[0:1] offset:1024 glc
@@ -3358,11 +3314,11 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX10-SDAG-NEXT: buffer_gl1_inv
; GFX10-SDAG-NEXT: buffer_gl0_inv
; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX10-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX10-SDAG-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX10-SDAG-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX10-SDAG-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-SDAG-NEXT: s_cbranch_execnz .LBB16_1
; GFX10-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX10-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX10-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x10
; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -3371,16 +3327,14 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
;
; GFX11-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_8:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-SDAG-NEXT: s_load_b32 s3, s[4:5], 0x8
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[0:1], 0x400
-; GFX11-SDAG-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-SDAG-NEXT: s_load_b32 s3, s[0:1], 0x400
+; GFX11-SDAG-NEXT: s_and_b32 s2, s2, 0xff
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: s_mov_b32 s3, 0
; GFX11-SDAG-NEXT: .LBB16_1: ; %atomicrmw.start
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
@@ -3388,19 +3342,19 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
; GFX11-SDAG-NEXT: v_mov_b16_e32 v1.h, 0
; GFX11-SDAG-NEXT: v_and_b16 v1.l, 0xff, v2.l
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_sub_nc_u16 v1.l, v1.l, s3 clamp
+; GFX11-SDAG-NEXT: v_sub_nc_u16 v1.l, v1.l, s2 clamp
; GFX11-SDAG-NEXT: v_and_or_b32 v1, 0xffffff00, v2, v1
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v1, v0, v[1:2], s[0:1] offset:1024 glc
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: buffer_gl1_inv
; GFX11-SDAG-NEXT: buffer_gl0_inv
; GFX11-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v1, v2
-; GFX11-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX11-SDAG-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX11-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-SDAG-NEXT: s_cbranch_execnz .LBB16_1
; GFX11-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
-; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s2
+; GFX11-SDAG-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x10
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -3451,51 +3405,48 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_8(ptr addrspa
define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr addrspace(1) %ptr, i8 %data) {
; GFX9-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_8:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX9-GISEL-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_mov_b64 s[4:5], 0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0xff
; GFX9-GISEL-NEXT: v_mov_b32_e32 v3, 0xffffff00
-; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: s_load_dword s5, s[0:1], 0x400
-; GFX9-GISEL-NEXT: s_lshl_b32 s4, s4, 8
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT: s_load_dword s3, s[0:1], 0x400
+; GFX9-GISEL-NEXT: s_lshl_b32 s2, s2, 8
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX9-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX9-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-GISEL-NEXT: v_lshlrev_b16_e32 v0, 8, v1
-; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v0, s4 clamp
+; GFX9-GISEL-NEXT: v_sub_u16_e64 v0, v0, s2 clamp
; GFX9-GISEL-NEXT: v_and_b32_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX9-GISEL-NEXT: v_and_or_b32 v0, v1, v3, v0
; GFX9-GISEL-NEXT: global_atomic_cmpswap v0, v4, v[0:1], s[0:1] offset:1024 glc
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: buffer_wbinvl1_vol
; GFX9-GISEL-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-GISEL-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-GISEL-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-GISEL-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-GISEL-NEXT: s_cbranch_execnz .LBB17_1
; GFX9-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX10-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_8:
; GFX10-GISEL: ; %bb.0:
-; GFX10-GISEL-NEXT: s_clause 0x1
-; GFX10-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
+; GFX10-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v3, 0xff
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: s_load_dword s2, s[0:1], 0x400
-; GFX10-GISEL-NEXT: s_lshl_b32 s3, s3, 8
+; GFX10-GISEL-NEXT: s_load_dword s3, s[0:1], 0x400
+; GFX10-GISEL-NEXT: s_lshl_b32 s2, s2, 8
; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-GISEL-NEXT: s_mov_b32 s3, 0
; GFX10-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-GISEL-NEXT: v_lshlrev_b16 v0, 8, v1
-; GFX10-GISEL-NEXT: v_sub_nc_u16 v0, v0, s3 clamp
+; GFX10-GISEL-NEXT: v_sub_nc_u16 v0, v0, s2 clamp
; GFX10-GISEL-NEXT: v_and_b32_sdwa v0, v0, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:DWORD
; GFX10-GISEL-NEXT: v_and_or_b32 v0, 0xffffff00, v1, v0
; GFX10-GISEL-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:1024 glc
@@ -3504,30 +3455,28 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX10-GISEL-NEXT: buffer_gl0_inv
; GFX10-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX10-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX10-GISEL-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-GISEL-NEXT: s_cbranch_execnz .LBB17_1
; GFX10-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-GISEL-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_8:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_clause 0x1
-; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-GISEL-NEXT: s_load_b32 s3, s[4:5], 0x8
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: s_load_b32 s2, s[0:1], 0x400
+; GFX11-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x400
; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-GISEL-NEXT: s_lshl_b32 s3, s3, 8
+; GFX11-GISEL-NEXT: s_lshl_b32 s2, s2, 8
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-GISEL-NEXT: s_mov_b32 s2, 0
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: s_mov_b32 s3, 0
; GFX11-GISEL-NEXT: .p2align 6
; GFX11-GISEL-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_lshlrev_b16 v0.l, 8, v1.l
-; GFX11-GISEL-NEXT: v_sub_nc_u16 v0.l, v0.l, s3 clamp
+; GFX11-GISEL-NEXT: v_sub_nc_u16 v0.l, v0.l, s2 clamp
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_lshrrev_b16 v0.l, 8, v0.l
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xff, v0
@@ -3539,9 +3488,9 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX11-GISEL-NEXT: buffer_gl0_inv
; GFX11-GISEL-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-GISEL-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX11-GISEL-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX11-GISEL-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-GISEL-NEXT: s_cbranch_execnz .LBB17_1
; GFX11-GISEL-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-GISEL-NEXT: s_endpgm
@@ -3581,47 +3530,44 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
;
; GFX9-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_8:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX9-SDAG-NEXT: s_load_dword s5, s[8:9], 0x8
-; GFX9-SDAG-NEXT: s_mov_b64 s[2:3], 0
-; GFX9-SDAG-NEXT: s_movk_i32 s4, 0xff00
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-SDAG-NEXT: s_mov_b64 s[4:5], 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_movk_i32 s3, 0xff00
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX9-SDAG-NEXT: s_load_dword s6, s[0:1], 0x400
-; GFX9-SDAG-NEXT: s_and_b32 s5, s5, 0xff
+; GFX9-SDAG-NEXT: s_and_b32 s2, s2, 0xff
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s6
; GFX9-SDAG-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX9-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-SDAG-NEXT: v_sub_u16_sdwa v0, v1, s5 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX9-SDAG-NEXT: v_and_or_b32 v0, v1, s4, v0
+; GFX9-SDAG-NEXT: v_sub_u16_sdwa v0, v1, s2 clamp dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX9-SDAG-NEXT: v_and_or_b32 v0, v1, s3, v0
; GFX9-SDAG-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:1024 glc
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: buffer_wbinvl1_vol
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-SDAG-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_cbranch_execnz .LBB17_1
; GFX9-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX10-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_8:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX10-SDAG-NEXT: s_load_dword s3, s[8:9], 0x8
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: s_load_dword s2, s[0:1], 0x400
-; GFX10-SDAG-NEXT: s_and_b32 s3, s3, 0xff
+; GFX10-SDAG-NEXT: s_load_dword s3, s[0:1], 0x400
+; GFX10-SDAG-NEXT: s_and_b32 s2, s2, 0xff
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-SDAG-NEXT: s_mov_b32 s3, 0
; GFX10-SDAG-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX10-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v1
-; GFX10-SDAG-NEXT: v_sub_nc_u16 v0, v0, s3 clamp
+; GFX10-SDAG-NEXT: v_sub_nc_u16 v0, v0, s2 clamp
; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX10-SDAG-NEXT: v_and_or_b32 v0, 0xffffff00, v1, v0
; GFX10-SDAG-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:1024 glc
@@ -3630,30 +3576,28 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX10-SDAG-NEXT: buffer_gl0_inv
; GFX10-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX10-SDAG-NEXT: v_mov_b32_e32 v1, v0
-; GFX10-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
-; GFX10-SDAG-NEXT: s_andn2_b32 exec_lo, exec_lo, s2
+; GFX10-SDAG-NEXT: s_or_b32 s3, vcc_lo, s3
+; GFX10-SDAG-NEXT: s_andn2_b32 exec_lo, exec_lo, s3
; GFX10-SDAG-NEXT: s_cbranch_execnz .LBB17_1
; GFX10-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX10-SDAG-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: global_atomic_usub_sat_sgpr_base_offset_nortn_8:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-SDAG-NEXT: s_load_b32 s3, s[4:5], 0x8
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[0:1], 0x400
-; GFX11-SDAG-NEXT: s_and_b32 s3, s3, 0xff
+; GFX11-SDAG-NEXT: s_load_b32 s3, s[0:1], 0x400
+; GFX11-SDAG-NEXT: s_and_b32 s2, s2, 0xff
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-SDAG-NEXT: s_mov_b32 s2, 0
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: s_mov_b32 s3, 0
; GFX11-SDAG-NEXT: .LBB17_1: ; %atomicrmw.start
; GFX11-SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_and_b16 v0.l, 0xff, v1.l
; GFX11-SDAG-NEXT: v_mov_b16_e32 v0.h, 0
-; GFX11-SDAG-NEXT: v_sub_nc_u16 v0.l, v0.l, s3 clamp
+; GFX11-SDAG-NEXT: v_sub_nc_u16 v0.l, v0.l, s2 clamp
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_and_or_b32 v0, 0xffffff00, v1, v0
; GFX11-SDAG-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:1024 glc
@@ -3662,9 +3606,9 @@ define amdgpu_kernel void @global_atomic_usub_sat_sgpr_base_offset_nortn_8(ptr a
; GFX11-SDAG-NEXT: buffer_gl0_inv
; GFX11-SDAG-NEXT: v_cmp_eq_u32_e32 vcc_lo, v0, v1
; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, v0
-; GFX11-SDAG-NEXT: s_or_b32 s2, vcc_lo, s2
+; GFX11-SDAG-NEXT: s_or_b32 s3, vcc_lo, s3
; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s2
+; GFX11-SDAG-NEXT: s_and_not1_b32 exec_lo, exec_lo, s3
; GFX11-SDAG-NEXT: s_cbranch_execnz .LBB17_1
; GFX11-SDAG-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX11-SDAG-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll
index 232103a7035f4..3a8180c4dbca3 100644
--- a/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomics-hw-remarks-gfx90a.ll
@@ -20,7 +20,7 @@ main_body:
}
; GFX90A-HW-LABEL: atomic_add_unsafe_hw_agent:
-; GFX90A-HW: global_atomic_add_f32 v0, v1, s[2:3]
+; GFX90A-HW: global_atomic_add_f32 v0, v1, s{{\[[0-9]+:[0-9]+\]}}
; GFX90A-HW: s_endpgm
define amdgpu_kernel void @atomic_add_unsafe_hw_agent(ptr addrspace(1) %ptr, float %val) #0 {
main_body:
@@ -29,7 +29,7 @@ main_body:
}
; GFX90A-HW-LABEL: atomic_add_unsafe_hw_wg:
-; GFX90A-HW: global_atomic_add_f32 v0, v1, s[2:3]
+; GFX90A-HW: global_atomic_add_f32 v0, v1, s{{\[[0-9]+:[0-9]+\]}}
; GFX90A-HW: s_endpgm
define amdgpu_kernel void @atomic_add_unsafe_hw_wg(ptr addrspace(1) %ptr, float %val) #0 {
main_body:
@@ -38,7 +38,7 @@ main_body:
}
; GFX90A-HW-LABEL: atomic_add_unsafe_hw_wavefront:
-; GFX90A-HW: global_atomic_add_f32 v0, v1, s[2:3]
+; GFX90A-HW: global_atomic_add_f32 v0, v1, s{{\[[0-9]+:[0-9]+\]}}
; GFX90A-HW: s_endpgm
define amdgpu_kernel void @atomic_add_unsafe_hw_wavefront(ptr addrspace(1) %ptr, float %val) #0 {
main_body:
@@ -47,7 +47,7 @@ main_body:
}
; GFX90A-HW-LABEL: atomic_add_unsafe_hw_single_thread:
-; GFX90A-HW: global_atomic_add_f32 v0, v1, s[2:3]
+; GFX90A-HW: global_atomic_add_f32 v0, v1, s{{\[[0-9]+:[0-9]+\]}}
; GFX90A-HW: s_endpgm
define amdgpu_kernel void @atomic_add_unsafe_hw_single_thread(ptr addrspace(1) %ptr, float %val) #0 {
main_body:
@@ -56,7 +56,7 @@ main_body:
}
; GFX90A-HW-LABEL: atomic_add_unsafe_hw_aoa:
-; GFX90A-HW: global_atomic_add_f32 v0, v1, s[2:3]
+; GFX90A-HW: global_atomic_add_f32 v0, v1, s{{\[[0-9]+:[0-9]+\]}}
; GFX90A-HW: s_endpgm
define amdgpu_kernel void @atomic_add_unsafe_hw_aoa(ptr addrspace(1) %ptr, float %val) #0 {
main_body:
@@ -65,7 +65,7 @@ main_body:
}
; GFX90A-HW-LABEL: atomic_add_unsafe_hw_wgoa:
-; GFX90A-HW: global_atomic_add_f32 v0, v1, s[2:3]
+; GFX90A-HW: global_atomic_add_f32 v0, v1, s{{\[[0-9]+:[0-9]+\]}}
; GFX90A-HW: s_endpgm
define amdgpu_kernel void @atomic_add_unsafe_hw_wgoa(ptr addrspace(1) %ptr, float %val) #0 {
main_body:
@@ -74,7 +74,7 @@ main_body:
}
; GFX90A-HW-LABEL: atomic_add_unsafe_hw_wfoa:
-; GFX90A-HW: global_atomic_add_f32 v0, v1, s[2:3]
+; GFX90A-HW: global_atomic_add_f32 v0, v1, s{{\[[0-9]+:[0-9]+\]}}
; GFX90A-HW: s_endpgm
define amdgpu_kernel void @atomic_add_unsafe_hw_wfoa(ptr addrspace(1) %ptr, float %val) #0 {
main_body:
@@ -83,7 +83,7 @@ main_body:
}
; GFX90A-HW-LABEL: atomic_add_unsafe_hw_stoa:
-; GFX90A-HW: global_atomic_add_f32 v0, v1, s[2:3]
+; GFX90A-HW: global_atomic_add_f32 v0, v1, s{{\[[0-9]+:[0-9]+\]}}
; GFX90A-HW: s_endpgm
define amdgpu_kernel void @atomic_add_unsafe_hw_stoa(ptr addrspace(1) %ptr, float %val) #0 {
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll b/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll
index d13611935b2fb..8744da9973b5b 100644
--- a/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomics_cond_sub.ll
@@ -182,11 +182,11 @@ entry:
define amdgpu_kernel void @global_atomic_usub_cond_rtn_u32(ptr addrspace(1) %addr, i32 %in, ptr addrspace(1) %use) {
; GFX12-SDAG-LABEL: global_atomic_usub_cond_rtn_u32:
; GFX12-SDAG: ; %bb.0: ; %entry
+; GFX12-SDAG-NEXT: s_clause 0x1
; GFX12-SDAG-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX12-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_mov_b32_e32 v1, s2
+; GFX12-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-SDAG-NEXT: global_wb scope:SCOPE_SYS
; GFX12-SDAG-NEXT: s_wait_storecnt 0x0
; GFX12-SDAG-NEXT: global_atomic_cond_sub_u32 v1, v0, v1, s[0:1] offset:16 th:TH_ATOMIC_RETURN scope:SCOPE_SYS
diff --git a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
index 8d2e1c8b89ffd..5f194277337c0 100644
--- a/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/av-split-dead-valno-crash.ll
@@ -4,23 +4,20 @@
define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2, i1 %cond.i.i.i2295, ptr addrspace(1) %ptr, ptr %ptr1) #0 {
; CHECK-LABEL: vgpr_mfma_pass_av_split_crash:
; CHECK: ; %bb.0: ; %entry
-; CHECK-NEXT: s_load_dword s0, s[4:5], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[10:11], s[4:5], 0x0
-; CHECK-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x10
+; CHECK-NEXT: s_load_dwordx8 s[12:19], s[4:5], 0x0
; CHECK-NEXT: v_mov_b32_e32 v30, 0x9037ab78
; CHECK-NEXT: v_mov_b32_e32 v31, 0x3e21eeb6
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_bitcmp1_b32 s0, 0
-; CHECK-NEXT: s_cselect_b64 s[16:17], -1, 0
-; CHECK-NEXT: s_xor_b64 s[18:19], s[16:17], -1
-; CHECK-NEXT: s_bitcmp1_b32 s0, 8
-; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; CHECK-NEXT: s_xor_b64 s[20:21], s[2:3], -1
-; CHECK-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v0
-; CHECK-NEXT: s_and_b64 s[2:3], exec, s[2:3]
; CHECK-NEXT: v_mov_b32_e32 v2, 0xa17f65f6
; CHECK-NEXT: v_mov_b32_e32 v3, 0xbe927e4f
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_bitcmp1_b32 s14, 0
+; CHECK-NEXT: s_cselect_b64 s[10:11], -1, 0
+; CHECK-NEXT: s_xor_b64 s[20:21], s[10:11], -1
+; CHECK-NEXT: s_bitcmp1_b32 s14, 8
+; CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; CHECK-NEXT: s_xor_b64 s[14:15], s[0:1], -1
+; CHECK-NEXT: s_and_b64 s[0:1], exec, s[0:1]
; CHECK-NEXT: v_mov_b32_e32 v4, 0x19f4ec90
; CHECK-NEXT: v_mov_b32_e32 v5, 0x3efa01a0
; CHECK-NEXT: v_mov_b32_e32 v6, 0x16c16967
@@ -29,30 +26,31 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_mov_b32_e32 v9, 0x3f4b2bb0
; CHECK-NEXT: v_mov_b32_e32 v10, 0xa57d9582
; CHECK-NEXT: v_mov_b32_e32 v11, 0xbf8c6ea4
+; CHECK-NEXT: v_cmp_ne_u32_e64 s[2:3], 1, v0
; CHECK-NEXT: v_mov_b32_e32 v12, 0xe82d3ff0
; CHECK-NEXT: v_mov_b32_e32 v13, 0xbfa59976
; CHECK-NEXT: v_mov_b32_e32 v14, 0x8427b883
; CHECK-NEXT: v_mov_b32_e32 v15, 0x3fae1bb4
-; CHECK-NEXT: s_mov_b64 s[22:23], 0
; CHECK-NEXT: v_mov_b32_e32 v16, 0x57b87036
; CHECK-NEXT: v_mov_b32_e32 v17, 0x3fb3b136
-; CHECK-NEXT: s_and_b64 s[4:5], exec, s[16:17]
+; CHECK-NEXT: s_mov_b64 s[22:23], 0
; CHECK-NEXT: v_mov_b32_e32 v18, 0x55555523
; CHECK-NEXT: v_mov_b32_e32 v19, 0xbfd55555
-; CHECK-NEXT: s_and_b64 s[6:7], exec, s[18:19]
+; CHECK-NEXT: s_and_b64 s[4:5], exec, s[10:11]
+; CHECK-NEXT: s_and_b64 s[6:7], exec, s[20:21]
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b64_e32 v[20:21], 0
; CHECK-NEXT: ; implicit-def: $agpr0_agpr1
; CHECK-NEXT: ; implicit-def: $vgpr22_vgpr23
; CHECK-NEXT: s_branch .LBB0_2
-; CHECK-NEXT: .LBB0_1: ; %Flow9
+; CHECK-NEXT: .LBB0_1: ; %Flow10
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_andn2_b64 vcc, exec, s[24:25]
; CHECK-NEXT: s_cbranch_vccz .LBB0_17
; CHECK-NEXT: .LBB0_2: ; %._crit_edge1942.i.i.i3548
; CHECK-NEXT: ; =>This Loop Header: Depth=1
; CHECK-NEXT: ; Child Loop BB0_6 Depth 2
-; CHECK-NEXT: s_and_b64 vcc, exec, s[0:1]
+; CHECK-NEXT: s_and_b64 vcc, exec, s[2:3]
; CHECK-NEXT: s_cbranch_vccnz .LBB0_9
; CHECK-NEXT: ; %bb.3: ; %.preheader1868.i.i.i3244
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
@@ -60,7 +58,7 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_cbranch_vccz .LBB0_10
; CHECK-NEXT: ; %bb.4: ; %.preheader1855.i.i.i3329.preheader
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[14:15]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[18:19]
; CHECK-NEXT: flat_load_dwordx2 v[24:25], v[24:25]
; CHECK-NEXT: v_mov_b64_e32 v[26:27], v[30:31]
; CHECK-NEXT: v_mov_b64_e32 v[28:29], v[2:3]
@@ -97,14 +95,14 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_accvgpr_read_b32 v26, a2
; CHECK-NEXT: s_mov_b64 s[24:25], -1
; CHECK-NEXT: s_mov_b64 s[8:9], -1
-; CHECK-NEXT: s_mov_b64 vcc, s[2:3]
+; CHECK-NEXT: s_mov_b64 vcc, s[0:1]
; CHECK-NEXT: ; implicit-def: $agpr2_agpr3
; CHECK-NEXT: s_cbranch_vccz .LBB0_5
; CHECK-NEXT: ; %bb.7: ; %.lr.ph2070.i.i.i3291
; CHECK-NEXT: ; in Loop: Header=BB0_6 Depth=2
; CHECK-NEXT: v_accvgpr_mov_b32 a3, a1
; CHECK-NEXT: v_accvgpr_mov_b32 a2, a0
-; CHECK-NEXT: s_mov_b64 s[8:9], s[18:19]
+; CHECK-NEXT: s_mov_b64 s[8:9], s[20:21]
; CHECK-NEXT: s_mov_b64 vcc, s[6:7]
; CHECK-NEXT: s_cbranch_vccz .LBB0_5
; CHECK-NEXT: ; %bb.8: ; %.preheader1856.preheader.i.i.i3325
@@ -115,11 +113,11 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: s_mov_b64 s[8:9], 0
; CHECK-NEXT: s_branch .LBB0_5
; CHECK-NEXT: .LBB0_9: ; in Loop: Header=BB0_2 Depth=1
-; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[10:11]
+; CHECK-NEXT: v_mov_b64_e32 v[24:25], s[12:13]
; CHECK-NEXT: v_accvgpr_write_b32 a0, v24
; CHECK-NEXT: s_mov_b64 s[22:23], 0
; CHECK-NEXT: v_accvgpr_write_b32 a1, v25
-; CHECK-NEXT: s_mov_b64 s[8:9], s[20:21]
+; CHECK-NEXT: s_mov_b64 s[8:9], s[14:15]
; CHECK-NEXT: s_branch .LBB0_15
; CHECK-NEXT: .LBB0_10: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[8:9], -1
@@ -133,12 +131,12 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: v_cmp_nlg_f64_e64 s[8:9], 0, v[26:27]
; CHECK-NEXT: v_accvgpr_write_b32 a0, v24
-; CHECK-NEXT: v_cndmask_b32_e64 v23, v23, 0, s[16:17]
+; CHECK-NEXT: v_cndmask_b32_e64 v23, v23, 0, s[10:11]
; CHECK-NEXT: v_cndmask_b32_e64 v26, 0, 1, s[8:9]
; CHECK-NEXT: v_mov_b32_e32 v27, v26
-; CHECK-NEXT: s_and_b64 s[8:9], exec, s[16:17]
-; CHECK-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[16:17]
-; CHECK-NEXT: global_store_dwordx2 v0, v[26:27], s[12:13]
+; CHECK-NEXT: s_and_b64 s[8:9], exec, s[10:11]
+; CHECK-NEXT: v_cndmask_b32_e64 v22, v22, 0, s[10:11]
+; CHECK-NEXT: global_store_dwordx2 v0, v[26:27], s[16:17]
; CHECK-NEXT: s_cselect_b32 s23, s23, 0
; CHECK-NEXT: s_cselect_b32 s22, s22, 0
; CHECK-NEXT: s_mov_b64 s[8:9], -1
@@ -147,10 +145,10 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: v_accvgpr_write_b32 a0, v24
; CHECK-NEXT: s_mov_b64 s[8:9], 0
; CHECK-NEXT: v_mov_b64_e32 v[22:23], 0
-; CHECK-NEXT: .LBB0_14: ; %Flow6
+; CHECK-NEXT: .LBB0_14: ; %Flow7
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: v_accvgpr_write_b32 a1, v25
-; CHECK-NEXT: .LBB0_15: ; %Flow6
+; CHECK-NEXT: .LBB0_15: ; %Flow7
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[24:25], -1
; CHECK-NEXT: s_and_b64 vcc, exec, s[8:9]
@@ -158,7 +156,7 @@ define amdgpu_kernel void @vgpr_mfma_pass_av_split_crash(double %arg1, i1 %arg2,
; CHECK-NEXT: ; %bb.16: ; %._crit_edge2105.i.i.i2330
; CHECK-NEXT: ; in Loop: Header=BB0_2 Depth=1
; CHECK-NEXT: s_mov_b64 s[24:25], 0
-; CHECK-NEXT: global_store_dwordx2 v0, v[20:21], s[12:13]
+; CHECK-NEXT: global_store_dwordx2 v0, v[20:21], s[16:17]
; CHECK-NEXT: s_branch .LBB0_1
; CHECK-NEXT: .LBB0_17: ; %DummyReturnBlock
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/bfe-combine.ll b/llvm/test/CodeGen/AMDGPU/bfe-combine.ll
index 3bf3e47f546f4..f04c2088ce33b 100644
--- a/llvm/test/CodeGen/AMDGPU/bfe-combine.ll
+++ b/llvm/test/CodeGen/AMDGPU/bfe-combine.ll
@@ -6,8 +6,7 @@
define amdgpu_kernel void @bfe_combine8(ptr addrspace(1) nocapture %arg, i32 %x) {
; VI-LABEL: bfe_combine8:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; VI-NEXT: v_bfe_u32 v0, v0, 8, 8
@@ -24,15 +23,14 @@ define amdgpu_kernel void @bfe_combine8(ptr addrspace(1) nocapture %arg, i32 %x)
;
; VI-SDWA-LABEL: bfe_combine8:
; VI-SDWA: ; %bb.0:
-; VI-SDWA-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDWA-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-SDWA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDWA-NEXT: v_mov_b32_e32 v1, 2
; VI-SDWA-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDWA-NEXT: v_add_u32_e32 v0, vcc, s2, v0
; VI-SDWA-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
-; VI-SDWA-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDWA-NEXT: v_mov_b32_e32 v2, s1
; VI-SDWA-NEXT: v_add_u32_e32 v0, vcc, s0, v0
-; VI-SDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
+; VI-SDWA-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
; VI-SDWA-NEXT: flat_load_dword v2, v[0:1]
; VI-SDWA-NEXT: v_mov_b32_e32 v0, s0
; VI-SDWA-NEXT: v_mov_b32_e32 v1, s1
@@ -42,21 +40,21 @@ define amdgpu_kernel void @bfe_combine8(ptr addrspace(1) nocapture %arg, i32 %x)
;
; CI-LABEL: bfe_combine8:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s6, 0
-; CI-NEXT: s_mov_b32 s7, s3
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, s7
+; CI-NEXT: v_mov_b32_e32 v1, 0
+; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: v_add_i32_e32 v0, vcc, s2, v0
; CI-NEXT: v_lshrrev_b32_e32 v0, 6, v0
-; CI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; CI-NEXT: s_mov_b32 s2, 0
; CI-NEXT: v_and_b32_e32 v0, 0x3fc, v0
-; CI-NEXT: v_mov_b32_e32 v1, 0
-; CI-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_mov_b32 s4, s0
+; CI-NEXT: s_mov_b32 s5, s1
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
%id = tail call i32 @llvm.amdgcn.workitem.id.x() #2
%idx = add i32 %x, %id
@@ -71,8 +69,7 @@ define amdgpu_kernel void @bfe_combine8(ptr addrspace(1) nocapture %arg, i32 %x)
define amdgpu_kernel void @bfe_combine16(ptr addrspace(1) nocapture %arg, i32 %x) {
; VI-LABEL: bfe_combine16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_add_u32_e32 v0, vcc, s2, v0
@@ -91,13 +88,12 @@ define amdgpu_kernel void @bfe_combine16(ptr addrspace(1) nocapture %arg, i32 %x
;
; VI-SDWA-LABEL: bfe_combine16:
; VI-SDWA: ; %bb.0:
-; VI-SDWA-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDWA-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDWA-NEXT: v_mov_b32_e32 v1, 15
+; VI-SDWA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDWA-NEXT: v_mov_b32_e32 v2, 15
+; VI-SDWA-NEXT: v_mov_b32_e32 v1, 0
; VI-SDWA-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDWA-NEXT: v_add_u32_e32 v0, vcc, s2, v0
-; VI-SDWA-NEXT: v_lshlrev_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; VI-SDWA-NEXT: v_mov_b32_e32 v1, 0
+; VI-SDWA-NEXT: v_lshlrev_b32_sdwa v0, v2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; VI-SDWA-NEXT: v_lshlrev_b64 v[0:1], 2, v[0:1]
; VI-SDWA-NEXT: v_mov_b32_e32 v2, s1
; VI-SDWA-NEXT: v_add_u32_e32 v0, vcc, s0, v0
@@ -111,22 +107,22 @@ define amdgpu_kernel void @bfe_combine16(ptr addrspace(1) nocapture %arg, i32 %x
;
; CI-LABEL: bfe_combine16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: v_mov_b32_e32 v1, 0
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s6, 0
+; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, s7
+; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: v_add_i32_e32 v0, vcc, s2, v0
; CI-NEXT: v_lshrrev_b32_e32 v0, 1, v0
; CI-NEXT: v_and_b32_e32 v0, 0x7fff8000, v0
; CI-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
-; CI-NEXT: s_mov_b32 s7, s3
-; CI-NEXT: s_mov_b64 s[4:5], s[0:1]
-; CI-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
+; CI-NEXT: s_mov_b32 s4, s0
+; CI-NEXT: s_mov_b32 s5, s1
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
%id = tail call i32 @llvm.amdgcn.workitem.id.x() #2
%idx = add i32 %x, %id
diff --git a/llvm/test/CodeGen/AMDGPU/bfe-patterns.ll b/llvm/test/CodeGen/AMDGPU/bfe-patterns.ll
index c0d5f8a9d1c3b..c14678cafc7a4 100644
--- a/llvm/test/CodeGen/AMDGPU/bfe-patterns.ll
+++ b/llvm/test/CodeGen/AMDGPU/bfe-patterns.ll
@@ -120,17 +120,17 @@ define amdgpu_kernel void @s_ubfe_sub_i32(ptr addrspace(1) %out, i32 %src, i32 %
; SI-LABEL: s_ubfe_sub_i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_sub_i32 s2, 32, s5
-; SI-NEXT: s_lshl_b32 s4, s4, s2
-; SI-NEXT: s_lshr_b32 s4, s4, s2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_sub_i32 s3, 32, s3
+; SI-NEXT: s_lshl_b32 s2, s2, s3
+; SI-NEXT: s_lshr_b32 s2, s2, s3
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_ubfe_sub_i32:
@@ -160,20 +160,20 @@ define amdgpu_kernel void @s_ubfe_sub_multi_use_shl_i32(ptr addrspace(1) %out, i
; SI-LABEL: s_ubfe_sub_multi_use_shl_i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: s_sub_i32 s3, 32, s5
-; SI-NEXT: s_lshl_b32 s4, s4, s3
-; SI-NEXT: s_lshr_b32 s5, s4, s3
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_sub_i32 s3, 32, s3
+; SI-NEXT: s_lshl_b32 s2, s2, s3
+; SI-NEXT: s_lshr_b32 s3, s2, s3
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s3
+; SI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_endpgm
;
@@ -322,17 +322,17 @@ define amdgpu_kernel void @s_sbfe_sub_i32(ptr addrspace(1) %out, i32 %src, i32 %
; SI-LABEL: s_sbfe_sub_i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_sub_i32 s2, 32, s5
-; SI-NEXT: s_lshl_b32 s4, s4, s2
-; SI-NEXT: s_ashr_i32 s4, s4, s2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_sub_i32 s3, 32, s3
+; SI-NEXT: s_lshl_b32 s2, s2, s3
+; SI-NEXT: s_ashr_i32 s2, s2, s3
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_sbfe_sub_i32:
@@ -362,20 +362,20 @@ define amdgpu_kernel void @s_sbfe_sub_multi_use_shl_i32(ptr addrspace(1) %out, i
; SI-LABEL: s_sbfe_sub_multi_use_shl_i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: s_sub_i32 s3, 32, s5
-; SI-NEXT: s_lshl_b32 s4, s4, s3
-; SI-NEXT: s_ashr_i32 s5, s4, s3
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: v_mov_b32_e32 v2, s5
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_sub_i32 s3, 32, s3
+; SI-NEXT: s_lshl_b32 s2, s2, s3
+; SI-NEXT: s_ashr_i32 s3, s2, s3
+; SI-NEXT: v_mov_b32_e32 v1, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s3
+; SI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/bfi_int.ll b/llvm/test/CodeGen/AMDGPU/bfi_int.ll
index d0669af25b850..85c0edd4679d1 100644
--- a/llvm/test/CodeGen/AMDGPU/bfi_int.ll
+++ b/llvm/test/CodeGen/AMDGPU/bfi_int.ll
@@ -11,28 +11,30 @@
define amdgpu_kernel void @s_bfi_def_i32(ptr addrspace(1) %out, i32 %x, i32 %y, i32 %z) {
; GFX7-LABEL: s_bfi_def_i32:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dword s8, s[4:5], 0xd
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_xor_b32 s1, s1, s2
-; GFX7-NEXT: s_and_b32 s0, s1, s0
-; GFX7-NEXT: s_xor_b32 s0, s0, s2
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_xor_b32 s0, s3, s8
+; GFX7-NEXT: s_and_b32 s0, s0, s2
+; GFX7-NEXT: s_xor_b32 s0, s0, s8
+; GFX7-NEXT: s_mov_b32 s5, s1
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: s_bfi_def_i32:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s4, s[4:5], 0x34
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b32 s1, s1, s2
-; GFX8-NEXT: s_and_b32 s0, s1, s0
-; GFX8-NEXT: s_xor_b32 s0, s0, s2
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_xor_b32 s0, s3, s4
+; GFX8-NEXT: s_and_b32 s0, s0, s2
+; GFX8-NEXT: s_xor_b32 s0, s0, s4
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
@@ -40,15 +42,15 @@ define amdgpu_kernel void @s_bfi_def_i32(ptr addrspace(1) %out, i32 %x, i32 %y,
; GFX10-LABEL: s_bfi_def_i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s4, s[4:5], 0x34
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_xor_b32 s1, s1, s2
-; GFX10-NEXT: s_and_b32 s0, s1, s0
-; GFX10-NEXT: s_xor_b32 s0, s0, s2
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-NEXT: s_xor_b32 s3, s3, s4
+; GFX10-NEXT: s_and_b32 s2, s3, s2
+; GFX10-NEXT: s_xor_b32 s2, s2, s4
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: s_bfi_def_i32:
@@ -130,28 +132,30 @@ entry:
define amdgpu_kernel void @s_bfi_sha256_ch(ptr addrspace(1) %out, i32 %x, i32 %y, i32 %z) {
; GFX7-LABEL: s_bfi_sha256_ch:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dword s8, s[4:5], 0xd
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_xor_b32 s1, s1, s2
-; GFX7-NEXT: s_and_b32 s0, s0, s1
-; GFX7-NEXT: s_xor_b32 s0, s2, s0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_xor_b32 s0, s3, s8
+; GFX7-NEXT: s_and_b32 s0, s2, s0
+; GFX7-NEXT: s_xor_b32 s0, s8, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: s_bfi_sha256_ch:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s4, s[4:5], 0x34
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b32 s1, s1, s2
-; GFX8-NEXT: s_and_b32 s0, s0, s1
-; GFX8-NEXT: s_xor_b32 s0, s2, s0
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_xor_b32 s0, s3, s4
+; GFX8-NEXT: s_and_b32 s0, s2, s0
+; GFX8-NEXT: s_xor_b32 s0, s4, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
@@ -159,15 +163,15 @@ define amdgpu_kernel void @s_bfi_sha256_ch(ptr addrspace(1) %out, i32 %x, i32 %y
; GFX10-LABEL: s_bfi_sha256_ch:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s4, s[4:5], 0x34
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_xor_b32 s1, s1, s2
-; GFX10-NEXT: s_and_b32 s0, s0, s1
-; GFX10-NEXT: s_xor_b32 s0, s2, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-NEXT: s_xor_b32 s3, s3, s4
+; GFX10-NEXT: s_and_b32 s2, s2, s3
+; GFX10-NEXT: s_xor_b32 s2, s4, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: s_bfi_sha256_ch:
@@ -454,30 +458,32 @@ entry:
define amdgpu_kernel void @s_bfi_sha256_ma(ptr addrspace(1) %out, i32 %x, i32 %y, i32 %z) {
; GFX7-LABEL: s_bfi_sha256_ma:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dword s8, s[4:5], 0xd
; GFX7-NEXT: s_mov_b32 s7, 0xf000
; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_and_b32 s3, s0, s2
-; GFX7-NEXT: s_or_b32 s0, s0, s2
-; GFX7-NEXT: s_and_b32 s0, s1, s0
-; GFX7-NEXT: s_or_b32 s0, s3, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: s_or_b32 s1, s2, s8
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_and_b32 s0, s2, s8
+; GFX7-NEXT: s_and_b32 s1, s3, s1
+; GFX7-NEXT: s_or_b32 s0, s0, s1
; GFX7-NEXT: v_mov_b32_e32 v0, s0
; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: s_bfi_sha256_ma:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s4, s[4:5], 0x34
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s3, s0, s2
-; GFX8-NEXT: s_or_b32 s0, s0, s2
-; GFX8-NEXT: s_and_b32 s0, s1, s0
-; GFX8-NEXT: s_or_b32 s0, s3, s0
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: v_mov_b32_e32 v1, s5
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: s_or_b32 s1, s2, s4
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_and_b32 s0, s2, s4
+; GFX8-NEXT: s_and_b32 s1, s3, s1
+; GFX8-NEXT: s_or_b32 s0, s0, s1
; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
@@ -485,16 +491,16 @@ define amdgpu_kernel void @s_bfi_sha256_ma(ptr addrspace(1) %out, i32 %x, i32 %y
; GFX10-LABEL: s_bfi_sha256_ma:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s4, s[4:5], 0x34
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_or_b32 s3, s0, s2
-; GFX10-NEXT: s_and_b32 s0, s0, s2
-; GFX10-NEXT: s_and_b32 s1, s1, s3
-; GFX10-NEXT: s_or_b32 s0, s0, s1
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-NEXT: s_or_b32 s5, s2, s4
+; GFX10-NEXT: s_and_b32 s2, s2, s4
+; GFX10-NEXT: s_and_b32 s3, s3, s5
+; GFX10-NEXT: s_or_b32 s2, s2, s3
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: s_bfi_sha256_ma:
diff --git a/llvm/test/CodeGen/AMDGPU/bfi_nested.ll b/llvm/test/CodeGen/AMDGPU/bfi_nested.ll
index 7326adae8cbcb..bd76f34832f0a 100644
--- a/llvm/test/CodeGen/AMDGPU/bfi_nested.ll
+++ b/llvm/test/CodeGen/AMDGPU/bfi_nested.ll
@@ -284,15 +284,16 @@ define amdgpu_kernel void @v_bfi_dont_applied_for_scalar_ops(ptr addrspace(1) %o
; GCN-LABEL: v_bfi_dont_applied_for_scalar_ops:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_and_b32 s2, s5, 0xffff0000
-; GCN-NEXT: s_and_b32 s4, s4, 0xffff
-; GCN-NEXT: s_or_b32 s4, s4, s2
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_and_b32 s3, s3, 0xffff0000
+; GCN-NEXT: s_and_b32 s2, s2, 0xffff
+; GCN-NEXT: s_or_b32 s2, s2, s3
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%shift = lshr i32 %b, 16
%tr = trunc i32 %shift to i16
diff --git a/llvm/test/CodeGen/AMDGPU/bfm.ll b/llvm/test/CodeGen/AMDGPU/bfm.ll
index 172e07f6b792c..3dd0d7e3ac8e3 100644
--- a/llvm/test/CodeGen/AMDGPU/bfm.ll
+++ b/llvm/test/CodeGen/AMDGPU/bfm.ll
@@ -6,23 +6,24 @@ define amdgpu_kernel void @s_bfm_pattern(ptr addrspace(1) %out, i32 %x, i32 %y)
; SI-LABEL: s_bfm_pattern:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_bfm_b32 s4, s4, s5
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_bfm_b32 s2, s2, s3
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_bfm_pattern:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfm_b32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_bfm_b32 s0, s2, s3
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%a = shl i32 1, %x
@@ -35,10 +36,9 @@ define amdgpu_kernel void @s_bfm_pattern(ptr addrspace(1) %out, i32 %x, i32 %y)
define amdgpu_kernel void @s_bfm_pattern_simple(ptr addrspace(1) %out, i32 %x) #0 {
; SI-LABEL: s_bfm_pattern_simple:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_bfm_b32 s4, s2, 0
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -47,13 +47,12 @@ define amdgpu_kernel void @s_bfm_pattern_simple(ptr addrspace(1) %out, i32 %x) #
;
; VI-LABEL: s_bfm_pattern_simple:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfm_b32 s2, s2, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_bfm_b32 s0, s2, 0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%a = shl i32 1, %x
diff --git a/llvm/test/CodeGen/AMDGPU/bitreverse.ll b/llvm/test/CodeGen/AMDGPU/bitreverse.ll
index 5e5e6a6b19284..294921faf2734 100644
--- a/llvm/test/CodeGen/AMDGPU/bitreverse.ll
+++ b/llvm/test/CodeGen/AMDGPU/bitreverse.ll
@@ -23,28 +23,30 @@ declare <4 x i64> @llvm.bitreverse.v4i64(<4 x i64>) #1
define amdgpu_kernel void @s_brev_i16(ptr addrspace(1) noalias %out, i16 %val) #0 {
; SI-LABEL: s_brev_i16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s4, s6
-; SI-NEXT: s_lshr_b32 s4, s4, 16
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_brev_b32 s0, s2
+; SI-NEXT: s_lshr_b32 s0, s0, 16
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; FLAT-LABEL: s_brev_i16:
; FLAT: ; %bb.0:
-; FLAT-NEXT: s_load_dword s6, s[4:5], 0x2c
-; FLAT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; FLAT-NEXT: s_mov_b32 s3, 0xf000
-; FLAT-NEXT: s_mov_b32 s2, -1
+; FLAT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; FLAT-NEXT: s_mov_b32 s7, 0xf000
+; FLAT-NEXT: s_mov_b32 s6, -1
; FLAT-NEXT: s_waitcnt lgkmcnt(0)
-; FLAT-NEXT: s_brev_b32 s4, s6
-; FLAT-NEXT: s_lshr_b32 s4, s4, 16
-; FLAT-NEXT: v_mov_b32_e32 v0, s4
-; FLAT-NEXT: buffer_store_short v0, off, s[0:3], 0
+; FLAT-NEXT: s_mov_b32 s4, s0
+; FLAT-NEXT: s_brev_b32 s0, s2
+; FLAT-NEXT: s_lshr_b32 s0, s0, 16
+; FLAT-NEXT: s_mov_b32 s5, s1
+; FLAT-NEXT: v_mov_b32_e32 v0, s0
+; FLAT-NEXT: buffer_store_short v0, off, s[4:7], 0
; FLAT-NEXT: s_endpgm
;
; GISEL-LABEL: s_brev_i16:
@@ -62,9 +64,7 @@ define amdgpu_kernel void @s_brev_i16(ptr addrspace(1) noalias %out, i16 %val) #
;
; GFX11-FLAT-LABEL: s_brev_i16:
; GFX11-FLAT: ; %bb.0:
-; GFX11-FLAT-NEXT: s_clause 0x1
-; GFX11-FLAT-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FLAT-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FLAT-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FLAT-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FLAT-NEXT: s_brev_b32 s2, s2
; GFX11-FLAT-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -213,26 +213,28 @@ define amdgpu_kernel void @v_brev_i16(ptr addrspace(1) noalias %out, ptr addrspa
define amdgpu_kernel void @s_brev_i32(ptr addrspace(1) noalias %out, i32 %val) #0 {
; SI-LABEL: s_brev_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s4, s6
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_brev_b32 s0, s2
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; FLAT-LABEL: s_brev_i32:
; FLAT: ; %bb.0:
-; FLAT-NEXT: s_load_dword s6, s[4:5], 0x2c
-; FLAT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; FLAT-NEXT: s_mov_b32 s3, 0xf000
-; FLAT-NEXT: s_mov_b32 s2, -1
+; FLAT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; FLAT-NEXT: s_mov_b32 s7, 0xf000
+; FLAT-NEXT: s_mov_b32 s6, -1
; FLAT-NEXT: s_waitcnt lgkmcnt(0)
-; FLAT-NEXT: s_brev_b32 s4, s6
-; FLAT-NEXT: v_mov_b32_e32 v0, s4
-; FLAT-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; FLAT-NEXT: s_mov_b32 s4, s0
+; FLAT-NEXT: s_brev_b32 s0, s2
+; FLAT-NEXT: s_mov_b32 s5, s1
+; FLAT-NEXT: v_mov_b32_e32 v0, s0
+; FLAT-NEXT: buffer_store_dword v0, off, s[4:7], 0
; FLAT-NEXT: s_endpgm
;
; GISEL-LABEL: s_brev_i32:
@@ -249,11 +251,9 @@ define amdgpu_kernel void @s_brev_i32(ptr addrspace(1) noalias %out, i32 %val) #
;
; GFX11-FLAT-LABEL: s_brev_i32:
; GFX11-FLAT: ; %bb.0:
-; GFX11-FLAT-NEXT: s_clause 0x1
-; GFX11-FLAT-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FLAT-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-FLAT-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-FLAT-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FLAT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FLAT-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-FLAT-NEXT: s_brev_b32 s2, s2
; GFX11-FLAT-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FLAT-NEXT: v_mov_b32_e32 v0, s2
@@ -370,29 +370,31 @@ define amdgpu_kernel void @s_brev_v2i32(ptr addrspace(1) noalias %out, <2 x i32>
; SI-LABEL: s_brev_v2i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_brev_b32 s5, s5
-; SI-NEXT: s_brev_b32 s4, s4
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_brev_b32 s0, s3
+; SI-NEXT: s_brev_b32 s1, s2
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: v_mov_b32_e32 v1, s0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; FLAT-LABEL: s_brev_v2i32:
; FLAT: ; %bb.0:
; FLAT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; FLAT-NEXT: s_mov_b32 s7, 0xf000
+; FLAT-NEXT: s_mov_b32 s6, -1
; FLAT-NEXT: s_waitcnt lgkmcnt(0)
-; FLAT-NEXT: s_mov_b64 s[4:5], s[2:3]
-; FLAT-NEXT: s_brev_b32 s5, s5
-; FLAT-NEXT: s_brev_b32 s4, s4
-; FLAT-NEXT: s_mov_b32 s3, 0xf000
-; FLAT-NEXT: s_mov_b32 s2, -1
-; FLAT-NEXT: v_mov_b32_e32 v0, s4
-; FLAT-NEXT: v_mov_b32_e32 v1, s5
-; FLAT-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; FLAT-NEXT: s_mov_b32 s4, s0
+; FLAT-NEXT: s_mov_b32 s5, s1
+; FLAT-NEXT: s_brev_b32 s0, s3
+; FLAT-NEXT: s_brev_b32 s1, s2
+; FLAT-NEXT: v_mov_b32_e32 v0, s1
+; FLAT-NEXT: v_mov_b32_e32 v1, s0
+; FLAT-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; FLAT-NEXT: s_endpgm
;
; GISEL-LABEL: s_brev_v2i32:
@@ -411,14 +413,16 @@ define amdgpu_kernel void @s_brev_v2i32(ptr addrspace(1) noalias %out, <2 x i32>
; GFX11-FLAT-LABEL: s_brev_v2i32:
; GFX11-FLAT: ; %bb.0:
; GFX11-FLAT-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-FLAT-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-FLAT-NEXT: s_mov_b32 s6, -1
; GFX11-FLAT-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FLAT-NEXT: s_brev_b32 s2, s2
; GFX11-FLAT-NEXT: s_brev_b32 s3, s3
; GFX11-FLAT-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FLAT-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX11-FLAT-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-FLAT-NEXT: s_mov_b32 s2, -1
-; GFX11-FLAT-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
+; GFX11-FLAT-NEXT: s_mov_b32 s4, s0
+; GFX11-FLAT-NEXT: s_mov_b32 s5, s1
+; GFX11-FLAT-NEXT: buffer_store_b64 v[0:1], off, s[4:7], 0
; GFX11-FLAT-NEXT: s_endpgm
;
; GFX11-GISEL-LABEL: s_brev_v2i32:
diff --git a/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll b/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
index 554d7eb2cfbdb..5a79c2ed7060d 100644
--- a/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
+++ b/llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll
@@ -9,7 +9,9 @@ define amdgpu_kernel void @blender_no_live_segment_at_def_error(<4 x float> %ext
; CHECK-NEXT: s_addc_u32 s13, s13, 0
; CHECK-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_LO), s12
; CHECK-NEXT: s_setreg_b32 hwreg(HW_REG_FLAT_SCR_HI), s13
+; CHECK-NEXT: s_clause 0x1
; CHECK-NEXT: s_load_dwordx8 s[48:55], s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx2 s[34:35], s[8:9], 0x20
; CHECK-NEXT: s_add_u32 s0, s0, s17
; CHECK-NEXT: s_addc_u32 s1, s1, 0
; CHECK-NEXT: s_mov_b32 s12, 0
@@ -47,28 +49,25 @@ define amdgpu_kernel void @blender_no_live_segment_at_def_error(<4 x float> %ext
; CHECK-NEXT: s_mov_b32 s51, s48
; CHECK-NEXT: s_cbranch_vccnz .LBB0_8
; CHECK-NEXT: ; %bb.7: ; %if.end273.i.i
-; CHECK-NEXT: s_add_u32 s18, s8, 40
-; CHECK-NEXT: s_addc_u32 s19, s9, 0
-; CHECK-NEXT: s_getpc_b64 s[20:21]
-; CHECK-NEXT: s_add_u32 s20, s20, _Z3dotDv3_fS_ at gotpcrel32@lo+4
-; CHECK-NEXT: s_addc_u32 s21, s21, _Z3dotDv3_fS_ at gotpcrel32@hi+12
+; CHECK-NEXT: s_add_u32 s8, s8, 40
+; CHECK-NEXT: s_addc_u32 s9, s9, 0
+; CHECK-NEXT: s_getpc_b64 s[18:19]
+; CHECK-NEXT: s_add_u32 s18, s18, _Z3dotDv3_fS_ at gotpcrel32@lo+4
+; CHECK-NEXT: s_addc_u32 s19, s19, _Z3dotDv3_fS_ at gotpcrel32@hi+12
; CHECK-NEXT: v_lshlrev_b32_e32 v2, 20, v2
-; CHECK-NEXT: s_load_dwordx2 s[20:21], s[20:21], 0x0
+; CHECK-NEXT: s_load_dwordx2 s[18:19], s[18:19], 0x0
; CHECK-NEXT: v_lshlrev_b32_e32 v3, 10, v1
; CHECK-NEXT: v_add_f32_e64 v1, s12, s15
-; CHECK-NEXT: s_mov_b64 s[34:35], s[8:9]
-; CHECK-NEXT: s_mov_b64 s[8:9], s[18:19]
; CHECK-NEXT: s_mov_b32 s12, s14
+; CHECK-NEXT: s_mov_b32 s14, s16
+; CHECK-NEXT: s_mov_b32 s48, 0
; CHECK-NEXT: v_or3_b32 v31, v0, v3, v2
; CHECK-NEXT: v_mov_b32_e32 v0, v1
; CHECK-NEXT: v_mov_b32_e32 v1, 0
; CHECK-NEXT: v_mov_b32_e32 v2, 0
-; CHECK-NEXT: s_mov_b32 s14, s16
-; CHECK-NEXT: s_mov_b32 s48, 0
; CHECK-NEXT: ; implicit-def: $sgpr15
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_swappc_b64 s[30:31], s[20:21]
-; CHECK-NEXT: s_mov_b64 s[8:9], s[34:35]
+; CHECK-NEXT: s_swappc_b64 s[30:31], s[18:19]
; CHECK-NEXT: s_mov_b32 s49, s48
; CHECK-NEXT: s_mov_b32 s50, s48
; CHECK-NEXT: s_mov_b32 s51, s48
@@ -79,14 +78,12 @@ define amdgpu_kernel void @blender_no_live_segment_at_def_error(<4 x float> %ext
; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], 0 offset:4
; CHECK-NEXT: buffer_store_dword v0, off, s[0:3], 0
; CHECK-NEXT: .LBB0_9: ; %kernel_direct_lighting.exit
-; CHECK-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x20
; CHECK-NEXT: v_mov_b32_e32 v0, s48
; CHECK-NEXT: v_mov_b32_e32 v4, 0
; CHECK-NEXT: v_mov_b32_e32 v1, s49
; CHECK-NEXT: v_mov_b32_e32 v2, s50
; CHECK-NEXT: v_mov_b32_e32 v3, s51
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_dwordx4 v4, v[0:3], s[4:5]
+; CHECK-NEXT: global_store_dwordx4 v4, v[0:3], s[34:35]
; CHECK-NEXT: s_endpgm
entry:
%cmp5.i.i = icmp eq i32 %cmp5.i.i.arg, 0
diff --git a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
index f9c7186831ee7..956d7837e56d1 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-folding-implicit-def-subreg.ll
@@ -14,80 +14,78 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: $sgpr1 = S_ADDC_U32 $sgpr1, 0, implicit-def dead $scc, implicit $scc, implicit-def $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: renamable $sgpr17 = COPY $sgpr15
; GFX90A-NEXT: renamable $vgpr31 = COPY $vgpr0, implicit $exec
- ; GFX90A-NEXT: early-clobber renamable $sgpr20_sgpr21_sgpr22_sgpr23 = S_LOAD_DWORDX4_IMM_ec renamable $sgpr8_sgpr9, 24, 0 :: (dereferenceable invariant load (s128) from %ir.arg3.kernarg.offset.align.down, align 8, addrspace 4)
- ; GFX90A-NEXT: renamable $sgpr24_sgpr25_sgpr26_sgpr27 = S_LOAD_DWORDX4_IMM renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
- ; GFX90A-NEXT: renamable $sgpr56_sgpr57 = S_LOAD_DWORDX2_IMM renamable $sgpr8_sgpr9, 16, 0 :: (dereferenceable invariant load (s64) from %ir.arg.kernarg.offset1 + 16, align 16, addrspace 4)
- ; GFX90A-NEXT: renamable $sgpr15 = S_LOAD_DWORD_IMM renamable $sgpr8_sgpr9, 40, 0 :: (dereferenceable invariant load (s32) from %ir.arg3.kernarg.offset.align.down + 16, align 8, addrspace 4)
- ; GFX90A-NEXT: S_BITCMP1_B32 renamable $sgpr20, 0, implicit-def $scc
+ ; GFX90A-NEXT: early-clobber renamable $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27 = S_LOAD_DWORDX8_IMM_ec renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s256) from %ir.arg.kernarg.offset1, align 16, addrspace 4)
+ ; GFX90A-NEXT: renamable $sgpr28_sgpr29_sgpr30_sgpr31 = S_LOAD_DWORDX4_IMM renamable $sgpr8_sgpr9, 32, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1 + 32, addrspace 4)
+ ; GFX90A-NEXT: S_BITCMP1_B32 renamable $sgpr26, 0, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr12_sgpr13 = S_CSELECT_B64 -1, 0, implicit killed $scc
; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_MOV_B64 -1
- ; GFX90A-NEXT: renamable $sgpr28_sgpr29 = S_XOR_B64 renamable $sgpr12_sgpr13, -1, implicit-def dead $scc
- ; GFX90A-NEXT: S_BITCMP1_B32 renamable $sgpr20, 8, implicit-def $scc
- ; GFX90A-NEXT: renamable $sgpr30_sgpr31 = S_CSELECT_B64 -1, 0, implicit killed $scc
- ; GFX90A-NEXT: renamable $sgpr30_sgpr31 = S_XOR_B64 killed renamable $sgpr30_sgpr31, -1, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr36_sgpr37 = S_XOR_B64 renamable $sgpr12_sgpr13, -1, implicit-def dead $scc
+ ; GFX90A-NEXT: S_BITCMP1_B32 renamable $sgpr26, 8, implicit-def $scc
+ ; GFX90A-NEXT: renamable $sgpr34_sgpr35 = S_CSELECT_B64 -1, 0, implicit killed $scc
+ ; GFX90A-NEXT: renamable $sgpr38_sgpr39 = S_XOR_B64 killed renamable $sgpr34_sgpr35, -1, implicit-def dead $scc
; GFX90A-NEXT: renamable $vgpr5 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr4 = DS_READ_B32_gfx9 renamable $vgpr5, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) null`, align 8, addrspace 3)
; GFX90A-NEXT: renamable $sgpr40_sgpr41 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, renamable $sgpr28_sgpr29, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, renamable $sgpr36_sgpr37, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_VCCZ %bb.2, implicit $vcc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.1.bb103:
- ; GFX90A-NEXT: successors: %bb.58(0x40000000), %bb.2(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr40_sgpr41, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr4_vgpr5:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: successors: %bb.59(0x40000000), %bb.2(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr28_sgpr29_sgpr30_sgpr31:0x000000000000003F, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFFF, $vgpr4_vgpr5:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, renamable $sgpr30_sgpr31, implicit-def dead $scc
- ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.58, implicit $vcc
+ ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, renamable $sgpr38_sgpr39, implicit-def dead $scc
+ ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.59, implicit $vcc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.2:
; GFX90A-NEXT: successors: %bb.3(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr40_sgpr41, $sgpr56, $sgpr57, $sgpr20_sgpr21_sgpr22, $sgpr22_sgpr23, $sgpr24_sgpr25_sgpr26, $sgpr26_sgpr27, $vgpr4, $vgpr5
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr28_sgpr29, $sgpr24_sgpr25_sgpr26_sgpr27, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26, $vgpr4, $vgpr5
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr17 = IMPLICIT_DEF implicit-def $vgpr16
; GFX90A-NEXT: renamable $vgpr3 = IMPLICIT_DEF implicit-def $vgpr2
; GFX90A-NEXT: renamable $vgpr27 = IMPLICIT_DEF implicit-def $vgpr26
; GFX90A-NEXT: renamable $vgpr29 = IMPLICIT_DEF implicit-def $vgpr28
; GFX90A-NEXT: renamable $vgpr33 = IMPLICIT_DEF implicit-def $vgpr32
- ; GFX90A-NEXT: renamable $sgpr36_sgpr37 = S_MOV_B64 0
+ ; GFX90A-NEXT: renamable $sgpr30_sgpr31 = S_MOV_B64 0
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.3.Flow17:
- ; GFX90A-NEXT: successors: %bb.4(0x40000000), %bb.57(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr40_sgpr41, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr16_vgpr17:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: successors: %bb.4(0x40000000), %bb.58(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFFF, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr16_vgpr17:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr6 = V_AND_B32_e32 1023, $vgpr31, implicit $exec
; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr18_sgpr19, implicit-def dead $scc
- ; GFX90A-NEXT: S_CBRANCH_VCCZ %bb.57, implicit $vcc
+ ; GFX90A-NEXT: S_CBRANCH_VCCZ %bb.58, implicit $vcc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.4.bb15:
; GFX90A-NEXT: successors: %bb.35(0x40000000), %bb.5(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFFF, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0_vgpr1 = nsw V_LSHLREV_B64_e64 2, $vgpr4_vgpr5, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr2 = COPY renamable $sgpr25, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr46, renamable $vcc = V_ADD_CO_U32_e64 $sgpr24, $vgpr0, 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr2 = COPY renamable $sgpr21, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr46, renamable $vcc = V_ADD_CO_U32_e64 $sgpr20, $vgpr0, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr47, dead renamable $vcc = V_ADDC_U32_e64 killed $vgpr2, killed $vgpr1, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr2 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 2, $vgpr6, implicit $exec
; GFX90A-NEXT: renamable $vgpr40, renamable $vcc = V_ADD_CO_U32_e64 $vgpr46, killed $vgpr0, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr41, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr47, killed $vcc, 0, implicit $exec
- ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, renamable $sgpr30_sgpr31, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, renamable $sgpr38_sgpr39, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.35, implicit $vcc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.5:
; GFX90A-NEXT: successors: %bb.6(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr34_sgpr35 = S_MOV_B64 -1
; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr44_sgpr45 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr42_sgpr43 = S_MOV_B64 0
+ ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $sgpr38_sgpr39 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
@@ -110,7 +108,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.6.Flow20:
; GFX90A-NEXT: successors: %bb.7(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr26 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr28 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
@@ -122,16 +120,16 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr29 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.7.Flow19:
- ; GFX90A-NEXT: successors: %bb.62(0x40000000), %bb.8(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: successors: %bb.63(0x40000000), %bb.8(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_MOV_B64 0
- ; GFX90A-NEXT: $sgpr18_sgpr19 = S_AND_SAVEEXEC_B64 $sgpr36_sgpr37, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.62, implicit $exec
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_MOV_B64 0
+ ; GFX90A-NEXT: $sgpr18_sgpr19 = S_AND_SAVEEXEC_B64 $sgpr30_sgpr31, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.63, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.8.Flow32:
; GFX90A-NEXT: successors: %bb.9(0x40000000), %bb.10(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr18_sgpr19, implicit-def $scc
; GFX90A-NEXT: $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr40_sgpr41, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -140,15 +138,15 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.9.bb89:
; GFX90A-NEXT: successors: %bb.10(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr13, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr12, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.10.Flow33:
; GFX90A-NEXT: successors: %bb.11(0x40000000), %bb.12(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
; GFX90A-NEXT: $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr46_sgpr47, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -157,15 +155,15 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.11.bb84:
; GFX90A-NEXT: successors: %bb.12(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr11, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr10, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.12.Flow34:
; GFX90A-NEXT: successors: %bb.13(0x40000000), %bb.14(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
; GFX90A-NEXT: $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr44_sgpr45, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -174,15 +172,15 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.13.bb79:
; GFX90A-NEXT: successors: %bb.14(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr42_sgpr43, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr9, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr8, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.14.Flow35:
; GFX90A-NEXT: successors: %bb.15(0x40000000), %bb.16(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr42_sgpr43, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr34_sgpr35, $sgpr42_sgpr43, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $vgpr0_vgpr1:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
; GFX90A-NEXT: $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $sgpr42_sgpr43, implicit-def $exec, implicit-def $scc, implicit $exec
@@ -191,7 +189,7 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.15.bb72:
; GFX90A-NEXT: successors: %bb.16(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $vgpr0_vgpr1:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr8 = S_ADD_U32 renamable $sgpr8, 48, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr9 = S_ADDC_U32 killed renamable $sgpr9, 0, implicit-def dead $scc, implicit killed $scc
@@ -202,122 +200,122 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: $sgpr13 = COPY killed renamable $sgpr17
; GFX90A-NEXT: $sgpr14 = COPY killed renamable $sgpr16
; GFX90A-NEXT: dead $sgpr30_sgpr31 = SI_CALL killed renamable $sgpr18_sgpr19, @f2, csr_amdgpu_gfx90ainsts, implicit $sgpr4_sgpr5, implicit $sgpr6_sgpr7, implicit $sgpr8_sgpr9, implicit $sgpr10_sgpr11, implicit $sgpr12, implicit $sgpr13, implicit $sgpr14, implicit undef $sgpr15, implicit $vgpr31, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit $vgpr0, implicit $vgpr1
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.16.Flow36:
; GFX90A-NEXT: successors: %bb.17(0x40000000), %bb.18(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr36_sgpr37, implicit-def $scc
- ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr66_sgpr67, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr68_sgpr69, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_EXECZ %bb.18, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.17.bb67:
; GFX90A-NEXT: successors: %bb.18(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr47, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr46, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.18.Flow37:
; GFX90A-NEXT: successors: %bb.19(0x40000000), %bb.20(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
- ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr64_sgpr65, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr66_sgpr67, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_EXECZ %bb.20, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.19.bb62:
; GFX90A-NEXT: successors: %bb.20(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr63, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr62, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.20.Flow38:
; GFX90A-NEXT: successors: %bb.21(0x40000000), %bb.22(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
- ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr54_sgpr55, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr64_sgpr65, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_EXECZ %bb.22, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.21.bb54:
; GFX90A-NEXT: successors: %bb.22(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr61, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr60, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.22.Flow39:
; GFX90A-NEXT: successors: %bb.23(0x40000000), %bb.24(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
- ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr52_sgpr53, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr54_sgpr55, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_EXECZ %bb.24, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.23.bb47:
; GFX90A-NEXT: successors: %bb.24(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr59, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr58, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.24.Flow40:
; GFX90A-NEXT: successors: %bb.25(0x40000000), %bb.26(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
- ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr50_sgpr51, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr52_sgpr53, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_EXECZ %bb.26, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.25.bb40:
; GFX90A-NEXT: successors: %bb.26(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr57, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr56, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.26.Flow41:
; GFX90A-NEXT: successors: %bb.27(0x40000000), %bb.28(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr48_sgpr49, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
- ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr48_sgpr49, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr50_sgpr51, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_EXECZ %bb.28, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.27.bb33:
; GFX90A-NEXT: successors: %bb.28(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr45, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr44, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.28.Flow42:
; GFX90A-NEXT: successors: %bb.34(0x40000000), %bb.29(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr48_sgpr49, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
- ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr38_sgpr39, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr48_sgpr49, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: renamable $sgpr4_sgpr5 = S_XOR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.34, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.29.Flow43:
; GFX90A-NEXT: successors: %bb.30(0x40000000), %bb.31(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
; GFX90A-NEXT: $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr34_sgpr35, implicit-def dead $scc
@@ -325,17 +323,17 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.30.bb19:
; GFX90A-NEXT: successors: %bb.31(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr41, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr40, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.31.Flow44:
; GFX90A-NEXT: successors: %bb.32(0x40000000), %bb.33(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr68_sgpr69, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr70_sgpr71, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr68_sgpr69, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr4_sgpr5 = S_AND_SAVEEXEC_B64 $sgpr70_sgpr71, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECZ %bb.33, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.32.UnifiedUnreachableBlock:
@@ -351,32 +349,32 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.34.bb26:
; GFX90A-NEXT: successors: %bb.29(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr68_sgpr69, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr4_sgpr5, $sgpr34_sgpr35, $sgpr70_sgpr71, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET renamable $vgpr43, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr42, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_OR_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_OR_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
; GFX90A-NEXT: S_BRANCH %bb.29
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.35.bb20:
; GFX90A-NEXT: successors: %bb.37(0x40000000), %bb.36(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFF0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0 = FLAT_LOAD_SBYTE renamable $vgpr40_vgpr41, 1024, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i23)
; GFX90A-NEXT: renamable $vgpr42 = V_ADD_CO_U32_e32 1024, $vgpr40, implicit-def $vcc, implicit $exec
; GFX90A-NEXT: renamable $sgpr34_sgpr35 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $sgpr38_sgpr39 = S_MOV_B64 -1
+ ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 -1
; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr44_sgpr45 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr42_sgpr43 = S_MOV_B64 0
+ ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vgpr43, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_LT_I16_e64 0, killed $vgpr0, implicit $exec
- ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 0
+ ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
@@ -395,24 +393,25 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
- ; GFX90A-NEXT: $sgpr24_sgpr25 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr20_sgpr21 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.37, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.36.Flow21:
; GFX90A-NEXT: successors: %bb.6(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr24_sgpr25, implicit-def $scc
+ ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr20_sgpr21, implicit-def $scc
; GFX90A-NEXT: S_BRANCH %bb.6
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.37.bb27:
; GFX90A-NEXT: successors: %bb.39(0x40000000), %bb.38(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr40_sgpr41, $sgpr46_sgpr47, $sgpr44_sgpr45, $sgpr64_sgpr65, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr66_sgpr67, $sgpr48_sgpr49
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFF0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr46_sgpr47, $sgpr44_sgpr45, $sgpr66_sgpr67, $sgpr64_sgpr65, $sgpr54_sgpr55, $sgpr68_sgpr69, $sgpr52_sgpr53
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0 = FLAT_LOAD_UBYTE renamable $vgpr40_vgpr41, 2048, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i30)
; GFX90A-NEXT: renamable $vgpr44 = V_ADD_CO_U32_e32 2048, $vgpr40, implicit-def $vcc, implicit $exec
+ ; GFX90A-NEXT: renamable $sgpr56_sgpr57 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr42_sgpr43 = S_MOV_B64 -1
- ; GFX90A-NEXT: renamable $sgpr58_sgpr59 = COPY renamable $sgpr36_sgpr37
+ ; GFX90A-NEXT: renamable $sgpr58_sgpr59 = COPY renamable $sgpr30_sgpr31
; GFX90A-NEXT: renamable $sgpr60_sgpr61 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr62_sgpr63 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vgpr45, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
@@ -434,42 +433,42 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
- ; GFX90A-NEXT: $sgpr38_sgpr39 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr40_sgpr41 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.39, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.38.Flow22:
; GFX90A-NEXT: successors: %bb.36(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr38_sgpr39, implicit-def $scc
- ; GFX90A-NEXT: renamable $sgpr38_sgpr39 = S_XOR_B64 $exec, -1, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr42_sgpr43, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr66_sgpr67, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr40_sgpr41, implicit-def $scc
+ ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_XOR_B64 $exec, -1, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr42_sgpr43, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_AND_B64 killed renamable $sgpr66_sgpr67, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr42_sgpr43 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr40_sgpr41 = S_AND_B64 killed renamable $sgpr40_sgpr41, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr36_sgpr37 = S_ANDN2_B64 killed renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr36_sgpr37 = S_OR_B64 killed renamable $sgpr36_sgpr37, killed renamable $sgpr56_sgpr57, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr40_sgpr41 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_ANDN2_B64 killed renamable $sgpr30_sgpr31, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr30_sgpr31 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr30_sgpr31 = S_OR_B64 killed renamable $sgpr24_sgpr25, killed renamable $sgpr30_sgpr31, implicit-def dead $scc
; GFX90A-NEXT: S_BRANCH %bb.36
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.39.bb34:
; GFX90A-NEXT: successors: %bb.41(0x40000000), %bb.40(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr46_sgpr47, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr66_sgpr67
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFF0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr46_sgpr47, $sgpr52_sgpr53, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr54_sgpr55, $sgpr66_sgpr67, $sgpr68_sgpr69
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0 = FLAT_LOAD_UBYTE renamable $vgpr40_vgpr41, 3072, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i37)
; GFX90A-NEXT: renamable $vgpr56 = V_ADD_CO_U32_e32 3072, $vgpr40, implicit-def $vcc, implicit $exec
+ ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr44_sgpr45 = S_MOV_B64 -1
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = COPY renamable $sgpr36_sgpr37
+ ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = COPY renamable $sgpr30_sgpr31
; GFX90A-NEXT: renamable $sgpr58_sgpr59 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vgpr57, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
@@ -486,39 +485,39 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
- ; GFX90A-NEXT: $sgpr40_sgpr41 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr42_sgpr43 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.41, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.40.Flow23:
; GFX90A-NEXT: successors: %bb.38(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr68_sgpr69, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr40_sgpr41, implicit-def $scc
+ ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr42_sgpr43, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr42_sgpr43 = S_XOR_B64 $exec, -1, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr62_sgpr63 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_AND_B64 killed renamable $sgpr44_sgpr45, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr62_sgpr63 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr44_sgpr45 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr40_sgpr41 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr48_sgpr49, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_ANDN2_B64 renamable $sgpr30_sgpr31, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr58_sgpr59 = S_OR_B64 killed renamable $sgpr24_sgpr25, killed renamable $sgpr48_sgpr49, implicit-def dead $scc
; GFX90A-NEXT: S_BRANCH %bb.38
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.41.bb41:
; GFX90A-NEXT: successors: %bb.46(0x40000000), %bb.42(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr52_sgpr53, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr54_sgpr55, $sgpr66_sgpr67, $sgpr68_sgpr69
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFF0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr54_sgpr55, $sgpr52_sgpr53, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr58 = V_ADD_CO_U32_e32 4096, $vgpr40, implicit-def $vcc, implicit $exec
; GFX90A-NEXT: renamable $vgpr1, dead renamable $sgpr18_sgpr19 = V_ADDC_U32_e64 0, $vgpr41, $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $vgpr58, implicit $exec
; GFX90A-NEXT: renamable $vgpr0 = FLAT_LOAD_UBYTE killed renamable $vgpr0_vgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i44)
; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_MOV_B64 -1
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = COPY renamable $sgpr36_sgpr37
+ ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = COPY renamable $sgpr30_sgpr31
; GFX90A-NEXT: renamable $vgpr59, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
@@ -536,34 +535,35 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
- ; GFX90A-NEXT: $sgpr42_sgpr43 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr44_sgpr45 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.46, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.42.Flow24:
; GFX90A-NEXT: successors: %bb.40(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr42_sgpr43, implicit-def $scc
+ ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr44_sgpr45, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr44_sgpr45 = S_XOR_B64 $exec, -1, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr62_sgpr63 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr42_sgpr43 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_ANDN2_B64 renamable $sgpr30_sgpr31, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_OR_B64 killed renamable $sgpr42_sgpr43, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_OR_B64 killed renamable $sgpr24_sgpr25, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
; GFX90A-NEXT: S_BRANCH %bb.40
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.43.bb55:
; GFX90A-NEXT: successors: %bb.48(0x40000000), %bb.44(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr46_sgpr47, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59, $sgpr48_sgpr49
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFF0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr58_sgpr59, $sgpr52_sgpr53
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: S_BITCMP1_B32 renamable $sgpr20, 16, implicit-def $scc
+ ; GFX90A-NEXT: S_BITCMP1_B32 renamable $sgpr26, 16, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_CSELECT_B64 -1, 0, implicit killed $scc
+ ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 -1
; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_XOR_B64 renamable $sgpr64_sgpr65, -1, implicit-def dead $scc
; GFX90A-NEXT: renamable $vgpr62 = V_ADD_CO_U32_e32 6144, $vgpr40, implicit-def $vcc, implicit $exec
; GFX90A-NEXT: renamable $vgpr63, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
@@ -572,9 +572,9 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.44:
; GFX90A-NEXT: successors: %bb.45(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr57, $vgpr62, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $vgpr40, $vgpr61, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22, $sgpr22_sgpr23, $sgpr24_sgpr25_sgpr26, $sgpr26_sgpr27, $vgpr56, $vgpr47, $vgpr2, $vgpr4, $vgpr5, $vgpr6, $vgpr46, $vgpr45, $vgpr44, $vgpr43, $vgpr42, $vgpr41, $vgpr58, $vgpr60, $vgpr63, $vgpr59
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr58_sgpr59, $sgpr52_sgpr53
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = COPY renamable $sgpr36_sgpr37
+ ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = COPY renamable $sgpr30_sgpr31
; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
@@ -592,23 +592,23 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.45.Flow26:
; GFX90A-NEXT: successors: %bb.47(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr62_sgpr63 = S_XOR_B64 $exec, -1, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr46_sgpr47, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_OR_B64 killed renamable $sgpr46_sgpr47, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr60_sgpr61, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr56_sgpr57, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_ANDN2_B64 renamable $sgpr30_sgpr31, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_OR_B64 killed renamable $sgpr24_sgpr25, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
; GFX90A-NEXT: S_BRANCH %bb.47
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.46.bb48:
; GFX90A-NEXT: successors: %bb.43(0x40000000), %bb.47(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr66_sgpr67, $sgpr58_sgpr59, $sgpr68_sgpr69, $sgpr64_sgpr65, $sgpr46_sgpr47, $sgpr54_sgpr55, $sgpr60_sgpr61
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFF0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr64_sgpr65, $sgpr52_sgpr53, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr58_sgpr59
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr60 = V_ADD_CO_U32_e32 5120, $vgpr40, implicit-def $vcc, implicit $exec
; GFX90A-NEXT: renamable $sgpr18_sgpr19 = COPY $vcc
@@ -616,10 +616,10 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr1, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr0 = FLAT_LOAD_UBYTE killed renamable $vgpr0_vgpr1, 1024, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i51)
; GFX90A-NEXT: renamable $sgpr62_sgpr63 = S_MOV_B64 -1
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = COPY renamable $sgpr36_sgpr37
- ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_MOV_B64 0
+ ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = COPY renamable $sgpr30_sgpr31
; GFX90A-NEXT: renamable $vgpr61, dead renamable $vcc = V_ADDC_U32_e64 0, $vgpr41, killed $sgpr18_sgpr19, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr0, implicit $exec
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
@@ -634,59 +634,70 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
- ; GFX90A-NEXT: $sgpr44_sgpr45 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: $sgpr46_sgpr47 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.43, implicit $exec
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.47.Flow25:
; GFX90A-NEXT: successors: %bb.42(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr58_sgpr59, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr70_sgpr71, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr44_sgpr45, implicit-def $scc
+ ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr46_sgpr47, implicit-def $scc
; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_XOR_B64 $exec, -1, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr62_sgpr63 = S_AND_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr58_sgpr59, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr66_sgpr67, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr62_sgpr63 = S_AND_B64 killed renamable $sgpr62_sgpr63, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr70_sgpr71, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr58_sgpr59 = S_AND_B64 killed renamable $sgpr68_sgpr69, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr66_sgpr67, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr64_sgpr65, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_AND_B64 killed renamable $sgpr48_sgpr49, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr44_sgpr45 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_ANDN2_B64 renamable $sgpr30_sgpr31, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_AND_B64 killed renamable $sgpr50_sgpr51, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_OR_B64 killed renamable $sgpr44_sgpr45, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_OR_B64 killed renamable $sgpr24_sgpr25, killed renamable $sgpr50_sgpr51, implicit-def dead $scc
; GFX90A-NEXT: S_BRANCH %bb.42
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.48.bb63:
; GFX90A-NEXT: successors: %bb.50(0x40000000), %bb.49(0x40000000)
- ; GFX90A-NEXT: liveins: $vcc, $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr56_sgpr57:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59, $sgpr48_sgpr49
+ ; GFX90A-NEXT: liveins: $vcc, $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr64_sgpr65, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFF0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr58_sgpr59, $sgpr52_sgpr53
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_MOV_B64 0
+ ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 0
; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.50, implicit $vcc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.49:
- ; GFX90A-NEXT: successors: %bb.44(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59
+ ; GFX90A-NEXT: successors: %bb.52(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr58_sgpr59, $sgpr52_sgpr53
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 -1
- ; GFX90A-NEXT: S_BRANCH %bb.44
+ ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 -1
+ ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = COPY renamable $sgpr30_sgpr31
+ ; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: renamable $vgpr0_vgpr1 = IMPLICIT_DEF
+ ; GFX90A-NEXT: S_BRANCH %bb.52
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.50.bb68:
- ; GFX90A-NEXT: successors: %bb.54(0x40000000), %bb.51(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr56_sgpr57:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr48_sgpr49, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59
+ ; GFX90A-NEXT: successors: %bb.55(0x40000000), %bb.51(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr64_sgpr65, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFF0, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr58_sgpr59, $sgpr52_sgpr53
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0 = nuw nsw V_LSHLREV_B32_e32 3, $vgpr6, implicit $exec
; GFX90A-NEXT: renamable $vgpr1 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
+ ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr18_sgpr19, implicit-def dead $scc
- ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.54, implicit $vcc
+ ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.55, implicit $vcc
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: bb.51:
- ; GFX90A-NEXT: successors: %bb.45(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61, $sgpr58_sgpr59
+ ; GFX90A-NEXT: successors: %bb.52(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr56_sgpr57, $sgpr60_sgpr61, $sgpr58_sgpr59
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 -1
- ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = COPY renamable $sgpr36_sgpr37
+ ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_MOV_B64 -1
+ ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = COPY renamable $sgpr30_sgpr31
; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
+ ; GFX90A-NEXT: {{ $}}
+ ; GFX90A-NEXT: bb.52.Flow26:
+ ; GFX90A-NEXT: successors: %bb.45(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr60, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8, $sgpr9, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr28_sgpr29_sgpr30, $sgpr24_sgpr25_sgpr26_sgpr27, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26, $vgpr0, $vgpr1, $vgpr2, $vgpr4, $vgpr5, $vgpr6, $vgpr8, $vgpr9, $vgpr10, $vgpr11, $vgpr12, $vgpr13, $vgpr61, $vgpr63, $vgpr62, $vgpr40, $vgpr41, $vgpr42, $vgpr43, $vgpr44, $vgpr45, $vgpr46, $vgpr47, $vgpr56, $vgpr57, $vgpr58, $vgpr59
+ ; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
@@ -698,22 +709,22 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
; GFX90A-NEXT: S_BRANCH %bb.45
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.52.bb80:
- ; GFX90A-NEXT: successors: %bb.59(0x40000000), %bb.53(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr56_sgpr57:0x000000000000000F, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.53.bb80:
+ ; GFX90A-NEXT: successors: %bb.60(0x40000000), %bb.54(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFF0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: dead renamable $sgpr15 = S_BFE_U32 renamable $sgpr20, 65560, implicit-def $scc
+ ; GFX90A-NEXT: dead renamable $sgpr15 = S_BFE_U32 renamable $sgpr26, 65560, implicit-def $scc
; GFX90A-NEXT: renamable $vgpr10 = V_ADD_CO_U32_e32 4096, $vgpr0, implicit-def $vcc, implicit $exec
; GFX90A-NEXT: renamable $vgpr11, dead renamable $sgpr18_sgpr19 = V_ADDC_U32_e64 0, 0, killed $vcc, 0, implicit $exec
- ; GFX90A-NEXT: S_CBRANCH_SCC0 %bb.59, implicit killed $scc
+ ; GFX90A-NEXT: S_CBRANCH_SCC0 %bb.60, implicit killed $scc
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.53:
- ; GFX90A-NEXT: successors: %bb.61(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.54:
+ ; GFX90A-NEXT: successors: %bb.62(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr62_sgpr63, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_MOV_B64 -1
- ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = COPY renamable $sgpr36_sgpr37
+ ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_MOV_B64 0
+ ; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_MOV_B64 -1
+ ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = COPY renamable $sgpr30_sgpr31
; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
@@ -724,17 +735,16 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
- ; GFX90A-NEXT: S_BRANCH %bb.61
+ ; GFX90A-NEXT: S_BRANCH %bb.62
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.54.bb73:
- ; GFX90A-NEXT: successors: %bb.52(0x40000000), %bb.55(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr56_sgpr57:0x000000000000000F, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003F, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr54_sgpr55, $sgpr60_sgpr61
+ ; GFX90A-NEXT: bb.55.bb73:
+ ; GFX90A-NEXT: successors: %bb.53(0x40000000), %bb.56(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr64_sgpr65, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFF0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr52_sgpr53, $sgpr56_sgpr57, $sgpr60_sgpr61
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr3 = FLAT_LOAD_UBYTE renamable $vgpr0_vgpr1, 2048, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i76)
; GFX90A-NEXT: renamable $vgpr8 = V_ADD_CO_U32_e32 2048, $vgpr0, implicit-def $vcc, implicit $exec
- ; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr58_sgpr59 = S_MOV_B64 -1
- ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = COPY renamable $sgpr36_sgpr37
+ ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = COPY renamable $sgpr30_sgpr31
; GFX90A-NEXT: renamable $vgpr9, dead renamable $sgpr18_sgpr19 = V_ADDC_U32_e64 0, 0, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr3, implicit $exec
; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
@@ -749,38 +759,38 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
; GFX90A-NEXT: $sgpr62_sgpr63 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.52, implicit $exec
+ ; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.53, implicit $exec
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.55.Flow29:
+ ; GFX90A-NEXT: bb.56.Flow29:
; GFX90A-NEXT: successors: %bb.45(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr58_sgpr59, $sgpr60_sgpr61, $sgpr62_sgpr63, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr62_sgpr63, implicit-def $scc
; GFX90A-NEXT: S_BRANCH %bb.45
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.56.bb90:
- ; GFX90A-NEXT: successors: %bb.60(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr54_sgpr55, $sgpr56_sgpr57:0x000000000000000F, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.57.bb90:
+ ; GFX90A-NEXT: successors: %bb.61(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr56_sgpr57, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000CFF0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr30 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed $sgpr64_sgpr65, implicit $exec
; GFX90A-NEXT: renamable $vgpr3 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr7 = COPY renamable $sgpr21, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr24_vgpr25 = DS_READ_B64_gfx9 killed renamable $vgpr7, 0, 0, implicit $exec :: (load (s64) from %ir.4, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr7 = COPY renamable $sgpr27, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr24_vgpr25 = DS_READ_B64_gfx9 killed renamable $vgpr7, 0, 0, implicit $exec :: (load (s64) from %ir.5, addrspace 3)
; GFX90A-NEXT: renamable $vgpr22_vgpr23 = DS_READ_B64_gfx9 killed renamable $vgpr3, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
- ; GFX90A-NEXT: renamable $vgpr3 = COPY renamable $sgpr22, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr3, 0, 0, implicit $exec :: (load (s64) from %ir.5, addrspace 3)
- ; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_LSHR_B64 killed renamable $sgpr56_sgpr57, 1, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $vgpr3 = COPY renamable $sgpr28, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr20_vgpr21 = DS_READ_B64_gfx9 killed renamable $vgpr3, 0, 0, implicit $exec :: (load (s64) from %ir.6, addrspace 3)
+ ; GFX90A-NEXT: renamable $sgpr18_sgpr19 = S_LSHR_B64 renamable $sgpr24_sgpr25, 1, implicit-def dead $scc
; GFX90A-NEXT: renamable $vgpr18_vgpr19 = V_LSHRREV_B64_e64 1, $vgpr24_vgpr25, implicit $exec
; GFX90A-NEXT: renamable $vgpr7 = V_CNDMASK_B32_e64 0, 0, 0, 1, $sgpr12_sgpr13, implicit $exec
; GFX90A-NEXT: renamable $vgpr14_vgpr15 = V_LSHRREV_B64_e64 1, $vgpr22_vgpr23, implicit $exec
- ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_XOR_B64 $exec, -1, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_OR_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_XOR_B64 $exec, -1, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_OR_B64 renamable $sgpr30_sgpr31, $exec, implicit-def dead $scc
; GFX90A-NEXT: renamable $vgpr16 = COPY renamable $vgpr22, implicit $exec
- ; GFX90A-NEXT: S_BRANCH %bb.60
+ ; GFX90A-NEXT: S_BRANCH %bb.61
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.57:
+ ; GFX90A-NEXT: bb.58:
; GFX90A-NEXT: successors: %bb.7(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr40_sgpr41, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr30_sgpr31, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr24 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr20 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
@@ -792,13 +802,13 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $sgpr46_sgpr47 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr44_sgpr45 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr42_sgpr43 = S_MOV_B64 0
+ ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr66_sgpr67 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr64_sgpr65 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr50_sgpr51 = S_MOV_B64 0
; GFX90A-NEXT: renamable $sgpr48_sgpr49 = S_MOV_B64 0
- ; GFX90A-NEXT: renamable $sgpr38_sgpr39 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vgpr12_vgpr13 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr10_vgpr11 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr8_vgpr9 = IMPLICIT_DEF
@@ -815,33 +825,33 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $sgpr34_sgpr35 = S_MOV_B64 0
; GFX90A-NEXT: S_BRANCH %bb.7
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.58.bb105:
+ ; GFX90A-NEXT: bb.59.bb105:
; GFX90A-NEXT: successors: %bb.3(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr15, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr40_sgpr41, $sgpr56_sgpr57:0x000000000000000F, $sgpr20_sgpr21_sgpr22_sgpr23:0x00000000000000FF, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000FF, $vgpr4_vgpr5:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr28_sgpr29_sgpr30_sgpr31:0x000000000000003F, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000FFFF, $vgpr4_vgpr5:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr0 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr28_vgpr29 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from `ptr addrspace(3) null`, addrspace 3)
- ; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr23, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr26_vgpr27 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.419, addrspace 3)
- ; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr21, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr2_vgpr3 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.4, addrspace 3)
- ; GFX90A-NEXT: renamable $vgpr0 = COPY killed renamable $sgpr15, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr16_vgpr17 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.420, addrspace 3)
- ; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr22, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr32_vgpr33 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.5, addrspace 3)
- ; GFX90A-NEXT: renamable $sgpr36_sgpr37 = S_MOV_B64 -1
+ ; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr29, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr26_vgpr27 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.423, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr27, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr2_vgpr3 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.5, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr30, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr16_vgpr17 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.424, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr0 = COPY renamable $sgpr28, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr32_vgpr33 = DS_READ_B64_gfx9 killed renamable $vgpr0, 0, 0, implicit $exec :: (load (s64) from %ir.6, addrspace 3)
+ ; GFX90A-NEXT: renamable $sgpr30_sgpr31 = S_MOV_B64 -1
; GFX90A-NEXT: S_BRANCH %bb.3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.59.bb85:
- ; GFX90A-NEXT: successors: %bb.56(0x40000000), %bb.60(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr56_sgpr57:0x000000000000000F, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.60.bb85:
+ ; GFX90A-NEXT: successors: %bb.57(0x40000000), %bb.61(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr62_sgpr63, $sgpr64_sgpr65, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000CFF0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr12 = V_OR_B32_e32 1, $vgpr10, implicit $exec
; GFX90A-NEXT: renamable $vgpr13 = COPY renamable $vgpr11, implicit $exec
; GFX90A-NEXT: renamable $vgpr3 = FLAT_LOAD_UBYTE renamable $vgpr12_vgpr13, 0, 0, implicit $exec, implicit $flat_scr :: (load (s8) from %ir.i86)
- ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_MOV_B64 -1
+ ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_MOV_B64 -1
; GFX90A-NEXT: renamable $vcc = V_CMP_EQ_U16_e64 0, killed $vgpr3, implicit $exec
- ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = COPY renamable $sgpr36_sgpr37
+ ; GFX90A-NEXT: renamable $sgpr66_sgpr67 = COPY renamable $sgpr30_sgpr31
; GFX90A-NEXT: renamable $vgpr7 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr14 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr22 = IMPLICIT_DEF
@@ -851,69 +861,69 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr21 = IMPLICIT_DEF implicit-def $vgpr20
; GFX90A-NEXT: renamable $sgpr18 = IMPLICIT_DEF
; GFX90A-NEXT: renamable $vgpr16 = IMPLICIT_DEF
- ; GFX90A-NEXT: $sgpr54_sgpr55 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
- ; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.56, implicit $exec
+ ; GFX90A-NEXT: $sgpr56_sgpr57 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
+ ; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.57, implicit $exec
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.60.Flow31:
- ; GFX90A-NEXT: successors: %bb.61(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr62_sgpr63, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.61.Flow31:
+ ; GFX90A-NEXT: successors: %bb.62(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr20_sgpr21, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr56_sgpr57, $sgpr62_sgpr63, $sgpr66_sgpr67, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr54_sgpr55, implicit-def $scc
- ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_MOV_B64 0
+ ; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr56_sgpr57, implicit-def $scc
+ ; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_MOV_B64 0
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.61.Flow30:
- ; GFX90A-NEXT: successors: %bb.55(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr24_sgpr25, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr62_sgpr63, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.62.Flow30:
+ ; GFX90A-NEXT: successors: %bb.56(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19:0x0000000000000003, $sgpr20_sgpr21, $sgpr24_sgpr25, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr62_sgpr63, $sgpr66_sgpr67, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x0000000000000003, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x0000000000000003, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $sgpr58_sgpr59 = S_XOR_B64 $exec, -1, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr52_sgpr53, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_ANDN2_B64 renamable $sgpr36_sgpr37, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr66_sgpr67, $exec, implicit-def dead $scc
- ; GFX90A-NEXT: renamable $sgpr52_sgpr53 = S_OR_B64 killed renamable $sgpr52_sgpr53, killed renamable $sgpr56_sgpr57, implicit-def dead $scc
- ; GFX90A-NEXT: S_BRANCH %bb.55
+ ; GFX90A-NEXT: renamable $sgpr60_sgpr61 = S_AND_B64 killed renamable $sgpr24_sgpr25, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr56_sgpr57 = S_AND_B64 killed renamable $sgpr54_sgpr55, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_ANDN2_B64 renamable $sgpr30_sgpr31, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_AND_B64 killed renamable $sgpr66_sgpr67, $exec, implicit-def dead $scc
+ ; GFX90A-NEXT: renamable $sgpr54_sgpr55 = S_OR_B64 killed renamable $sgpr24_sgpr25, killed renamable $sgpr54_sgpr55, implicit-def dead $scc
+ ; GFX90A-NEXT: S_BRANCH %bb.56
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.62.bb140:
- ; GFX90A-NEXT: successors: %bb.68(0x40000000), %bb.63(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr30_sgpr31, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.63.bb140:
+ ; GFX90A-NEXT: successors: %bb.69(0x40000000), %bb.64(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_MOV_B64 -1
- ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr30_sgpr31, implicit-def dead $scc
- ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.68, implicit $vcc
+ ; GFX90A-NEXT: renamable $sgpr20_sgpr21 = S_MOV_B64 -1
+ ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr38_sgpr39, implicit-def dead $scc
+ ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.69, implicit $vcc
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.63.Flow13:
- ; GFX90A-NEXT: successors: %bb.64(0x40000000), %bb.66(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr24_sgpr25, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.64.Flow13:
+ ; GFX90A-NEXT: successors: %bb.65(0x40000000), %bb.67(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr20_sgpr21, $sgpr34_sgpr35, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr24_sgpr25, implicit-def dead $scc
- ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.66, implicit $vcc
+ ; GFX90A-NEXT: $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr20_sgpr21, implicit-def dead $scc
+ ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.67, implicit $vcc
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.64.bb159:
- ; GFX90A-NEXT: successors: %bb.67(0x40000000), %bb.65(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.65.bb159:
+ ; GFX90A-NEXT: successors: %bb.68(0x40000000), %bb.66(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vcc = V_CMP_NE_U32_e64 0, killed $vgpr6, implicit $exec
; GFX90A-NEXT: $sgpr12_sgpr13 = S_AND_SAVEEXEC_B64 $vcc, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: renamable $sgpr12_sgpr13 = S_XOR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
- ; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.67, implicit $exec
+ ; GFX90A-NEXT: S_CBRANCH_EXECNZ %bb.68, implicit $exec
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.65.Flow10:
- ; GFX90A-NEXT: successors: %bb.66(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.66.Flow10:
+ ; GFX90A-NEXT: successors: %bb.67(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $sgpr12_sgpr13 = S_ANDN2_SAVEEXEC_B64 $sgpr12_sgpr13, implicit-def $exec, implicit-def $scc, implicit $exec
; GFX90A-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def $scc
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.66.Flow14:
+ ; GFX90A-NEXT: bb.67.Flow14:
; GFX90A-NEXT: successors: %bb.8(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $sgpr68_sgpr69 = COPY $exec
+ ; GFX90A-NEXT: renamable $sgpr70_sgpr71 = COPY $exec
; GFX90A-NEXT: S_BRANCH %bb.8
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.67.bb161:
- ; GFX90A-NEXT: successors: %bb.65(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.68.bb161:
+ ; GFX90A-NEXT: successors: %bb.66(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr27, killed $vgpr29, implicit $exec
; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr33, implicit $exec
@@ -928,11 +938,11 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr2 = V_CNDMASK_B32_e64 0, 0, 0, killed $vgpr2, killed $vcc, implicit $exec
; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 killed $vgpr2, killed $vgpr14, implicit $exec
; GFX90A-NEXT: DS_WRITE2_B32_gfx9 killed renamable $vgpr3, killed renamable $vgpr2, renamable $vgpr3, 0, 1, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, align 4, addrspace 3)
- ; GFX90A-NEXT: S_BRANCH %bb.65
+ ; GFX90A-NEXT: S_BRANCH %bb.66
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.68.bb174:
- ; GFX90A-NEXT: successors: %bb.72(0x40000000), %bb.69(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr28_sgpr29, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.69.bb174:
+ ; GFX90A-NEXT: successors: %bb.73(0x40000000), %bb.70(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr36_sgpr37, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000F, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000F, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000F, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000F, $vgpr28_vgpr29:0x000000000000000F, $vgpr32_vgpr33:0x000000000000000F, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $agpr0 = COPY killed renamable $vgpr14, implicit $exec
; GFX90A-NEXT: renamable $vgpr34 = V_OR_B32_e32 1, $vgpr32, implicit $exec
@@ -944,23 +954,23 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr38 = V_OR_B32_e32 $vgpr50, $vgpr20, implicit $exec
; GFX90A-NEXT: renamable $vgpr14 = V_CNDMASK_B32_e64 0, 0, 0, $vgpr38, killed $sgpr12_sgpr13, implicit $exec
; GFX90A-NEXT: renamable $sgpr12_sgpr13 = S_MOV_B64 -1
- ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr28_sgpr29, implicit-def dead $scc
- ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.72, implicit $vcc
+ ; GFX90A-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr36_sgpr37, implicit-def dead $scc
+ ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.73, implicit $vcc
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.69.Flow:
- ; GFX90A-NEXT: successors: %bb.70(0x40000000), %bb.71(0x40000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $agpr0_agpr1:0x0000000000000003, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.70.Flow:
+ ; GFX90A-NEXT: successors: %bb.71(0x40000000), %bb.72(0x40000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $agpr0_agpr1:0x0000000000000003, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr12_sgpr13, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr12_sgpr13, implicit-def dead $scc
- ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.71, implicit $vcc
+ ; GFX90A-NEXT: S_CBRANCH_VCCNZ %bb.72, implicit $vcc
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.70.bb186:
- ; GFX90A-NEXT: successors: %bb.71(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $agpr0_agpr1:0x0000000000000003, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.71.bb186:
+ ; GFX90A-NEXT: successors: %bb.72(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $agpr0_agpr1:0x0000000000000003, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr4_vgpr5 = nsw V_LSHLREV_B64_e64 3, killed $vgpr4_vgpr5, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr2 = COPY renamable $sgpr27, implicit $exec
- ; GFX90A-NEXT: renamable $vgpr4, renamable $vcc = V_ADD_CO_U32_e64 killed $sgpr26, $vgpr4, 0, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr2 = COPY renamable $sgpr23, implicit $exec
+ ; GFX90A-NEXT: renamable $vgpr4, renamable $vcc = V_ADD_CO_U32_e64 $sgpr22, $vgpr4, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr2, dead renamable $vcc = V_ADDC_U32_e64 killed $vgpr2, killed $vgpr5, killed $vcc, 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr35 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: renamable $vgpr55 = COPY renamable $vgpr35, implicit $exec
@@ -971,36 +981,36 @@ define amdgpu_kernel void @f1(ptr addrspace(1) %arg, ptr addrspace(1) %arg1, i64
; GFX90A-NEXT: renamable $vgpr15 = COPY renamable $vgpr35, implicit $exec
; GFX90A-NEXT: renamable $vgpr39 = COPY renamable $vgpr35, implicit $exec
; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr35, renamable $vgpr34_vgpr35, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
- ; GFX90A-NEXT: renamable $vgpr5 = COPY renamable $sgpr21, implicit $exec
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr5, killed renamable $vgpr54_vgpr55, 0, 0, implicit $exec :: (store (s64) into %ir.4, addrspace 3)
- ; GFX90A-NEXT: renamable $vgpr16 = COPY killed renamable $sgpr22, implicit $exec
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr16, killed renamable $vgpr48_vgpr49, 0, 0, implicit $exec :: (store (s64) into %ir.5, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr5 = COPY killed renamable $sgpr27, implicit $exec
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr5, killed renamable $vgpr54_vgpr55, 0, 0, implicit $exec :: (store (s64) into %ir.5, addrspace 3)
+ ; GFX90A-NEXT: renamable $vgpr16 = COPY killed renamable $sgpr28, implicit $exec
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr16, killed renamable $vgpr48_vgpr49, 0, 0, implicit $exec :: (store (s64) into %ir.6, addrspace 3)
; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr35, killed renamable $vgpr52_vgpr53, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr5, killed renamable $vgpr50_vgpr51, 0, 0, implicit $exec :: (store (s64) into %ir.4, addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr5, killed renamable $vgpr50_vgpr51, 0, 0, implicit $exec :: (store (s64) into %ir.5, addrspace 3)
; GFX90A-NEXT: DS_WRITE_B64_gfx9 renamable $vgpr35, killed renamable $vgpr36_vgpr37, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
- ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr5, killed renamable $vgpr14_vgpr15, 0, 0, implicit $exec :: (store (s64) into %ir.4, addrspace 3)
+ ; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr5, killed renamable $vgpr14_vgpr15, 0, 0, implicit $exec :: (store (s64) into %ir.5, addrspace 3)
; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr35, killed renamable $vgpr38_vgpr39, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr2, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 4, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null` + 4, basealign 8, addrspace 5)
; GFX90A-NEXT: BUFFER_STORE_DWORD_OFFSET killed renamable $vgpr4, $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(5) null`, align 8, addrspace 5)
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.71.Flow9:
- ; GFX90A-NEXT: successors: %bb.63(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $agpr0_agpr1:0x0000000000000003, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.72.Flow9:
+ ; GFX90A-NEXT: successors: %bb.64(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $agpr0_agpr1:0x0000000000000003, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: renamable $sgpr24_sgpr25 = S_MOV_B64 0
+ ; GFX90A-NEXT: renamable $sgpr20_sgpr21 = S_MOV_B64 0
; GFX90A-NEXT: renamable $vgpr14 = COPY killed renamable $agpr0, implicit $exec
- ; GFX90A-NEXT: S_BRANCH %bb.63
+ ; GFX90A-NEXT: S_BRANCH %bb.64
; GFX90A-NEXT: {{ $}}
- ; GFX90A-NEXT: bb.72.bb196:
- ; GFX90A-NEXT: successors: %bb.69(0x80000000)
- ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $agpr0_agpr1:0x0000000000000003, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr38_sgpr39, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr20_sgpr21_sgpr22_sgpr23:0x000000000000003C, $sgpr24_sgpr25_sgpr26_sgpr27:0x00000000000000F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
+ ; GFX90A-NEXT: bb.73.bb196:
+ ; GFX90A-NEXT: successors: %bb.70(0x80000000)
+ ; GFX90A-NEXT: liveins: $sgpr14, $sgpr16, $sgpr17, $vgpr7, $vgpr30, $vgpr31, $agpr0_agpr1:0x0000000000000003, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9:0x000000000000000F, $sgpr10_sgpr11, $sgpr18_sgpr19, $sgpr34_sgpr35, $sgpr40_sgpr41, $sgpr42_sgpr43, $sgpr44_sgpr45, $sgpr46_sgpr47, $sgpr48_sgpr49, $sgpr50_sgpr51, $sgpr52_sgpr53, $sgpr54_sgpr55, $sgpr64_sgpr65, $sgpr66_sgpr67, $sgpr68_sgpr69, $sgpr28_sgpr29_sgpr30_sgpr31:0x0000000000000003, $sgpr20_sgpr21_sgpr22_sgpr23_sgpr24_sgpr25_sgpr26_sgpr27:0x000000000000C0F0, $vgpr0_vgpr1:0x000000000000000F, $vgpr2_vgpr3:0x000000000000000C, $vgpr4_vgpr5:0x000000000000000F, $vgpr6_vgpr7:0x0000000000000003, $vgpr8_vgpr9:0x000000000000000F, $vgpr10_vgpr11:0x000000000000000F, $vgpr12_vgpr13:0x000000000000000F, $vgpr14_vgpr15:0x0000000000000003, $vgpr16_vgpr17:0x000000000000000C, $vgpr18_vgpr19:0x0000000000000003, $vgpr20_vgpr21:0x000000000000000C, $vgpr22_vgpr23:0x0000000000000003, $vgpr24_vgpr25:0x0000000000000003, $vgpr26_vgpr27:0x000000000000000C, $vgpr28_vgpr29:0x000000000000000C, $vgpr32_vgpr33:0x000000000000000C, $vgpr34_vgpr35:0x0000000000000003, $vgpr36_vgpr37:0x0000000000000003, $vgpr38_vgpr39:0x0000000000000003, $vgpr40_vgpr41:0x000000000000000F, $vgpr42_vgpr43:0x000000000000000F, $vgpr44_vgpr45:0x000000000000000F, $vgpr46_vgpr47:0x000000000000000F, $vgpr48_vgpr49:0x0000000000000003, $vgpr50_vgpr51:0x0000000000000003, $vgpr52_vgpr53:0x0000000000000003, $vgpr54_vgpr55:0x0000000000000003, $vgpr56_vgpr57:0x000000000000000F, $vgpr58_vgpr59:0x000000000000000F, $vgpr60_vgpr61:0x000000000000000F, $vgpr62_vgpr63:0x000000000000000F, $sgpr0_sgpr1_sgpr2_sgpr3
; GFX90A-NEXT: {{ $}}
; GFX90A-NEXT: renamable $vgpr2 = V_OR_B32_e32 $vgpr14, killed $vgpr24, implicit $exec
; GFX90A-NEXT: renamable $vgpr22 = V_OR_B32_e32 killed $vgpr2, killed $vgpr22, implicit $exec
; GFX90A-NEXT: renamable $vgpr23 = AV_MOV_B32_IMM_PSEUDO 0, implicit $exec
; GFX90A-NEXT: DS_WRITE_B64_gfx9 killed renamable $vgpr23, renamable $vgpr22_vgpr23, 0, 0, implicit $exec :: (store (s64) into `ptr addrspace(3) null`, addrspace 3)
; GFX90A-NEXT: renamable $sgpr12_sgpr13 = S_MOV_B64 0
- ; GFX90A-NEXT: S_BRANCH %bb.69
+ ; GFX90A-NEXT: S_BRANCH %bb.70
bb:
%i = tail call i32 @llvm.amdgcn.workitem.id.x()
%i11 = icmp eq i32 %i, 0
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll
index e3273165906e9..59f9ecb78ff3a 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-relaxation-gfx1250.ll
@@ -24,16 +24,16 @@ define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addr
; GCN-LABEL: uniform_conditional_max_short_forward_branch:
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GCN-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: s_cmp_eq_u32 s0, 0
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
; GCN-NEXT: s_cbranch_scc0 .LBB0_1
; GCN-NEXT: ; %bb.3: ; %bb
-; GCN-NEXT: s_get_pc_i64 s[2:3]
+; GCN-NEXT: s_get_pc_i64 s[4:5]
; GCN-NEXT: .Lpost_getpc0:
-; GCN-NEXT: s_add_co_u32 s2, s2, (.LBB0_2-.Lpost_getpc0)&4294967295
-; GCN-NEXT: s_add_co_ci_u32 s3, s3, (.LBB0_2-.Lpost_getpc0)>>32
-; GCN-NEXT: s_set_pc_i64 s[2:3]
+; GCN-NEXT: s_add_co_u32 s4, s4, (.LBB0_2-.Lpost_getpc0)&4294967295
+; GCN-NEXT: s_add_co_ci_u32 s5, s5, (.LBB0_2-.Lpost_getpc0)>>32
+; GCN-NEXT: s_set_pc_i64 s[4:5]
; GCN-NEXT: .LBB0_1: ; %bb2
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: v_nop_e64
@@ -41,19 +41,17 @@ define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addr
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: s_sleep 0
; GCN-NEXT: .LBB0_2: ; %bb3
-; GCN-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GCN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GCN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GCN-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GCN-NEXT: s_wait_storecnt 0x0
; GCN-NEXT: s_endpgm
;
; GCN-ADD-PC64-LABEL: uniform_conditional_max_short_forward_branch:
; GCN-ADD-PC64: ; %bb.0: ; %bb
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-ADD-PC64-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GCN-ADD-PC64-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
-; GCN-ADD-PC64-NEXT: s_cmp_eq_u32 s0, 0
+; GCN-ADD-PC64-NEXT: s_cmp_eq_u32 s2, 0
; GCN-ADD-PC64-NEXT: s_cbranch_scc0 .LBB0_1
; GCN-ADD-PC64-NEXT: ; %bb.3: ; %bb
; GCN-ADD-PC64-NEXT: s_add_pc_i64 .LBB0_2-.Lpost_addpc0
@@ -65,10 +63,8 @@ define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addr
; GCN-ADD-PC64-NEXT: ;;#ASMEND
; GCN-ADD-PC64-NEXT: s_sleep 0
; GCN-ADD-PC64-NEXT: .LBB0_2: ; %bb3
-; GCN-ADD-PC64-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GCN-ADD-PC64-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
-; GCN-ADD-PC64-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GCN-ADD-PC64-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GCN-ADD-PC64-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GCN-ADD-PC64-NEXT: s_wait_storecnt 0x0
; GCN-ADD-PC64-NEXT: s_endpgm
; GCN-ENABLE-ADD-PC64-LABEL: uniform_conditional_max_short_forward_branch:
@@ -115,16 +111,16 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrs
; GCN-LABEL: uniform_conditional_min_long_forward_branch:
; GCN: ; %bb.0: ; %bb0
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GCN-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: s_cmp_eq_u32 s0, 0
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
; GCN-NEXT: s_cbranch_scc0 .LBB1_1
; GCN-NEXT: ; %bb.3: ; %bb0
-; GCN-NEXT: s_get_pc_i64 s[2:3]
+; GCN-NEXT: s_get_pc_i64 s[4:5]
; GCN-NEXT: .Lpost_getpc1:
-; GCN-NEXT: s_add_co_u32 s2, s2, (.LBB1_2-.Lpost_getpc1)&4294967295
-; GCN-NEXT: s_add_co_ci_u32 s3, s3, (.LBB1_2-.Lpost_getpc1)>>32
-; GCN-NEXT: s_set_pc_i64 s[2:3]
+; GCN-NEXT: s_add_co_u32 s4, s4, (.LBB1_2-.Lpost_getpc1)&4294967295
+; GCN-NEXT: s_add_co_ci_u32 s5, s5, (.LBB1_2-.Lpost_getpc1)>>32
+; GCN-NEXT: s_set_pc_i64 s[4:5]
; GCN-NEXT: .LBB1_1: ; %bb2
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: v_nop_e64
@@ -133,19 +129,17 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrs
; GCN-NEXT: s_sleep 0
; GCN-NEXT: s_sleep 0
; GCN-NEXT: .LBB1_2: ; %bb3
-; GCN-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GCN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GCN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GCN-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GCN-NEXT: s_wait_storecnt 0x0
; GCN-NEXT: s_endpgm
;
; GCN-ADD-PC64-LABEL: uniform_conditional_min_long_forward_branch:
; GCN-ADD-PC64: ; %bb.0: ; %bb0
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-ADD-PC64-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GCN-ADD-PC64-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
-; GCN-ADD-PC64-NEXT: s_cmp_eq_u32 s0, 0
+; GCN-ADD-PC64-NEXT: s_cmp_eq_u32 s2, 0
; GCN-ADD-PC64-NEXT: s_cbranch_scc0 .LBB1_1
; GCN-ADD-PC64-NEXT: ; %bb.3: ; %bb0
; GCN-ADD-PC64-NEXT: s_add_pc_i64 .LBB1_2-.Lpost_addpc1
@@ -158,10 +152,8 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrs
; GCN-ADD-PC64-NEXT: s_sleep 0
; GCN-ADD-PC64-NEXT: s_sleep 0
; GCN-ADD-PC64-NEXT: .LBB1_2: ; %bb3
-; GCN-ADD-PC64-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GCN-ADD-PC64-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
-; GCN-ADD-PC64-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GCN-ADD-PC64-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GCN-ADD-PC64-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GCN-ADD-PC64-NEXT: s_wait_storecnt 0x0
; GCN-ADD-PC64-NEXT: s_endpgm
; GCN-ENABLE-ADD-PC64-LABEL: uniform_conditional_min_long_forward_branch:
@@ -210,16 +202,16 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr
; GCN-LABEL: uniform_conditional_min_long_forward_vcnd_branch:
; GCN: ; %bb.0: ; %bb0
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GCN-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: s_cmp_eq_f32 s0, 0
+; GCN-NEXT: s_cmp_eq_f32 s2, 0
; GCN-NEXT: s_cbranch_scc0 .LBB2_1
; GCN-NEXT: ; %bb.3: ; %bb0
-; GCN-NEXT: s_get_pc_i64 s[2:3]
+; GCN-NEXT: s_get_pc_i64 s[4:5]
; GCN-NEXT: .Lpost_getpc2:
-; GCN-NEXT: s_add_co_u32 s2, s2, (.LBB2_2-.Lpost_getpc2)&4294967295
-; GCN-NEXT: s_add_co_ci_u32 s3, s3, (.LBB2_2-.Lpost_getpc2)>>32
-; GCN-NEXT: s_set_pc_i64 s[2:3]
+; GCN-NEXT: s_add_co_u32 s4, s4, (.LBB2_2-.Lpost_getpc2)&4294967295
+; GCN-NEXT: s_add_co_ci_u32 s5, s5, (.LBB2_2-.Lpost_getpc2)>>32
+; GCN-NEXT: s_set_pc_i64 s[4:5]
; GCN-NEXT: .LBB2_1: ; %bb2
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: ; 32 bytes
@@ -229,19 +221,17 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr
; GCN-NEXT: s_sleep 0
; GCN-NEXT: s_sleep 0
; GCN-NEXT: .LBB2_2: ; %bb3
-; GCN-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GCN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GCN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GCN-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GCN-NEXT: s_wait_storecnt 0x0
; GCN-NEXT: s_endpgm
;
; GCN-ADD-PC64-LABEL: uniform_conditional_min_long_forward_vcnd_branch:
; GCN-ADD-PC64: ; %bb.0: ; %bb0
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-ADD-PC64-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GCN-ADD-PC64-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
-; GCN-ADD-PC64-NEXT: s_cmp_eq_f32 s0, 0
+; GCN-ADD-PC64-NEXT: s_cmp_eq_f32 s2, 0
; GCN-ADD-PC64-NEXT: s_cbranch_scc0 .LBB2_1
; GCN-ADD-PC64-NEXT: ; %bb.3: ; %bb0
; GCN-ADD-PC64-NEXT: s_add_pc_i64 .LBB2_2-.Lpost_addpc2
@@ -255,10 +245,8 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr
; GCN-ADD-PC64-NEXT: s_sleep 0
; GCN-ADD-PC64-NEXT: s_sleep 0
; GCN-ADD-PC64-NEXT: .LBB2_2: ; %bb3
-; GCN-ADD-PC64-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
-; GCN-ADD-PC64-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
-; GCN-ADD-PC64-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GCN-ADD-PC64-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GCN-ADD-PC64-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GCN-ADD-PC64-NEXT: s_wait_storecnt 0x0
; GCN-ADD-PC64-NEXT: s_endpgm
; GCN-ENABLE-ADD-PC64-LABEL: uniform_conditional_min_long_forward_vcnd_branch:
@@ -511,28 +499,27 @@ define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr add
; GCN-LABEL: uniform_unconditional_min_long_forward_branch:
; GCN: ; %bb.0: ; %bb0
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GCN-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: s_cmp_eq_u32 s0, 0
-; GCN-NEXT: s_mov_b32 s0, -1
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-NEXT: s_mov_b32 s2, -1
; GCN-NEXT: s_cbranch_scc0 .LBB5_1
; GCN-NEXT: ; %bb.7: ; %bb0
-; GCN-NEXT: s_get_pc_i64 s[0:1]
+; GCN-NEXT: s_get_pc_i64 s[2:3]
; GCN-NEXT: .Lpost_getpc6:
-; GCN-NEXT: s_add_co_u32 s0, s0, (.LBB5_4-.Lpost_getpc6)&4294967295
-; GCN-NEXT: s_add_co_ci_u32 s1, s1, (.LBB5_4-.Lpost_getpc6)>>32
-; GCN-NEXT: s_set_pc_i64 s[0:1]
+; GCN-NEXT: s_add_co_u32 s2, s2, (.LBB5_4-.Lpost_getpc6)&4294967295
+; GCN-NEXT: s_add_co_ci_u32 s3, s3, (.LBB5_4-.Lpost_getpc6)>>32
+; GCN-NEXT: s_set_pc_i64 s[2:3]
; GCN-NEXT: .LBB5_1: ; %Flow
-; GCN-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
+; GCN-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
; GCN-NEXT: s_cbranch_vccnz .LBB5_3
; GCN-NEXT: .LBB5_2: ; %bb2
; GCN-NEXT: v_mov_b32_e32 v0, 17
; GCN-NEXT: global_store_b32 v[0:1], v0, off scope:SCOPE_SYS
; GCN-NEXT: s_wait_storecnt 0x0
; GCN-NEXT: .LBB5_3: ; %bb4
-; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GCN-NEXT: s_wait_xcnt 0x0
; GCN-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 63
-; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GCN-NEXT: s_wait_storecnt 0x0
; GCN-NEXT: s_endpgm
@@ -545,40 +532,39 @@ define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr add
; GCN-NEXT: s_sleep 0
; GCN-NEXT: s_cbranch_execnz .LBB5_5
; GCN-NEXT: ; %bb.9: ; %bb3
-; GCN-NEXT: s_get_pc_i64 s[0:1]
+; GCN-NEXT: s_get_pc_i64 s[2:3]
; GCN-NEXT: .Lpost_getpc7:
-; GCN-NEXT: s_add_co_u32 s0, s0, (.LBB5_2-.Lpost_getpc7)&4294967295
-; GCN-NEXT: s_add_co_ci_u32 s1, s1, (.LBB5_2-.Lpost_getpc7)>>32
-; GCN-NEXT: s_set_pc_i64 s[0:1]
+; GCN-NEXT: s_add_co_u32 s2, s2, (.LBB5_2-.Lpost_getpc7)&4294967295
+; GCN-NEXT: s_add_co_ci_u32 s3, s3, (.LBB5_2-.Lpost_getpc7)>>32
+; GCN-NEXT: s_set_pc_i64 s[2:3]
; GCN-NEXT: .LBB5_5: ; %bb3
-; GCN-NEXT: s_get_pc_i64 s[0:1]
+; GCN-NEXT: s_get_pc_i64 s[2:3]
; GCN-NEXT: .Lpost_getpc5:
-; GCN-NEXT: s_add_co_u32 s0, s0, (.LBB5_3-.Lpost_getpc5)&4294967295
-; GCN-NEXT: s_add_co_ci_u32 s1, s1, (.LBB5_3-.Lpost_getpc5)>>32
-; GCN-NEXT: s_set_pc_i64 s[0:1]
+; GCN-NEXT: s_add_co_u32 s2, s2, (.LBB5_3-.Lpost_getpc5)&4294967295
+; GCN-NEXT: s_add_co_ci_u32 s3, s3, (.LBB5_3-.Lpost_getpc5)>>32
+; GCN-NEXT: s_set_pc_i64 s[2:3]
;
; GCN-ADD-PC64-LABEL: uniform_unconditional_min_long_forward_branch:
; GCN-ADD-PC64: ; %bb.0: ; %bb0
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-ADD-PC64-NEXT: s_load_b32 s0, s[4:5], 0x2c nv
+; GCN-ADD-PC64-NEXT: s_load_b96 s[0:2], s[4:5], 0x24 nv
; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
-; GCN-ADD-PC64-NEXT: s_cmp_eq_u32 s0, 0
-; GCN-ADD-PC64-NEXT: s_mov_b32 s0, -1
+; GCN-ADD-PC64-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-ADD-PC64-NEXT: s_mov_b32 s2, -1
; GCN-ADD-PC64-NEXT: s_cbranch_scc0 .LBB5_1
; GCN-ADD-PC64-NEXT: ; %bb.7: ; %bb0
; GCN-ADD-PC64-NEXT: s_add_pc_i64 .LBB5_4-.Lpost_addpc6
; GCN-ADD-PC64-NEXT: .Lpost_addpc6:
; GCN-ADD-PC64-NEXT: .LBB5_1: ; %Flow
-; GCN-ADD-PC64-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
+; GCN-ADD-PC64-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
; GCN-ADD-PC64-NEXT: s_cbranch_vccnz .LBB5_3
; GCN-ADD-PC64-NEXT: .LBB5_2: ; %bb2
; GCN-ADD-PC64-NEXT: v_mov_b32_e32 v0, 17
; GCN-ADD-PC64-NEXT: global_store_b32 v[0:1], v0, off scope:SCOPE_SYS
; GCN-ADD-PC64-NEXT: s_wait_storecnt 0x0
; GCN-ADD-PC64-NEXT: .LBB5_3: ; %bb4
-; GCN-ADD-PC64-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GCN-ADD-PC64-NEXT: s_wait_xcnt 0x0
; GCN-ADD-PC64-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 63
-; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
; GCN-ADD-PC64-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GCN-ADD-PC64-NEXT: s_wait_storecnt 0x0
; GCN-ADD-PC64-NEXT: s_endpgm
@@ -1151,48 +1137,48 @@ define amdgpu_kernel void @long_branch_hang(ptr addrspace(1) nocapture %arg, i32
; GCN-LABEL: long_branch_hang:
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c nv
-; GCN-NEXT: s_mov_b32 s7, -1
+; GCN-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
+; GCN-NEXT: s_mov_b32 s1, 0
+; GCN-NEXT: s_mov_b32 s2, -1
; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: s_cmp_eq_u32 s0, 0
-; GCN-NEXT: s_cselect_b32 s6, -1, 0
-; GCN-NEXT: s_cmp_lg_u32 s0, 0
-; GCN-NEXT: s_mov_b32 s0, 0
-; GCN-NEXT: s_cselect_b32 s8, -1, 0
-; GCN-NEXT: s_cmp_lt_i32 s3, 6
+; GCN-NEXT: s_cmp_eq_u32 s10, 0
+; GCN-NEXT: s_cselect_b32 s0, -1, 0
+; GCN-NEXT: s_cmp_lg_u32 s10, 0
+; GCN-NEXT: s_cselect_b32 s3, -1, 0
+; GCN-NEXT: s_cmp_lt_i32 s13, 6
; GCN-NEXT: s_cbranch_scc0 .LBB10_1
; GCN-NEXT: ; %bb.10: ; %bb
-; GCN-NEXT: s_get_pc_i64 s[10:11]
+; GCN-NEXT: s_get_pc_i64 s[4:5]
; GCN-NEXT: .Lpost_getpc14:
-; GCN-NEXT: s_add_co_u32 s10, s10, (.LBB10_4-.Lpost_getpc14)&4294967295
-; GCN-NEXT: s_add_co_ci_u32 s11, s11, (.LBB10_4-.Lpost_getpc14)>>32
-; GCN-NEXT: s_set_pc_i64 s[10:11]
+; GCN-NEXT: s_add_co_u32 s4, s4, (.LBB10_4-.Lpost_getpc14)&4294967295
+; GCN-NEXT: s_add_co_ci_u32 s5, s5, (.LBB10_4-.Lpost_getpc14)>>32
+; GCN-NEXT: s_set_pc_i64 s[4:5]
; GCN-NEXT: .LBB10_1: ; %Flow
-; GCN-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s7
+; GCN-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
; GCN-NEXT: s_cbranch_vccnz .LBB10_2
; GCN-NEXT: ; %bb.12: ; %Flow
-; GCN-NEXT: s_get_pc_i64 s[8:9]
+; GCN-NEXT: s_get_pc_i64 s[2:3]
; GCN-NEXT: .Lpost_getpc15:
-; GCN-NEXT: s_add_co_u32 s8, s8, (.LBB10_5-.Lpost_getpc15)&4294967295
-; GCN-NEXT: s_add_co_ci_u32 s9, s9, (.LBB10_5-.Lpost_getpc15)>>32
-; GCN-NEXT: s_set_pc_i64 s[8:9]
-; GCN-NEXT: .LBB10_2: ; %Flow5
-; GCN-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
+; GCN-NEXT: s_add_co_u32 s2, s2, (.LBB10_5-.Lpost_getpc15)&4294967295
+; GCN-NEXT: s_add_co_ci_u32 s3, s3, (.LBB10_5-.Lpost_getpc15)>>32
+; GCN-NEXT: s_set_pc_i64 s[2:3]
+; GCN-NEXT: .LBB10_2: ; %Flow7
+; GCN-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
; GCN-NEXT: s_cbranch_vccz .LBB10_3
-; GCN-NEXT: ; %bb.14: ; %Flow5
+; GCN-NEXT: ; %bb.14: ; %Flow7
; GCN-NEXT: s_get_pc_i64 s[0:1]
; GCN-NEXT: .Lpost_getpc16:
; GCN-NEXT: s_add_co_u32 s0, s0, (.LBB10_6-.Lpost_getpc16)&4294967295
; GCN-NEXT: s_add_co_ci_u32 s1, s1, (.LBB10_6-.Lpost_getpc16)>>32
; GCN-NEXT: s_set_pc_i64 s[0:1]
; GCN-NEXT: .LBB10_3: ; %bb14
-; GCN-NEXT: s_cmp_lt_i32 s1, 9
-; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: s_cmp_lt_i32 s2, s3
+; GCN-NEXT: s_cmp_lt_i32 s11, 9
; GCN-NEXT: s_cselect_b32 s1, -1, 0
+; GCN-NEXT: s_cmp_lt_i32 s12, s13
+; GCN-NEXT: s_cselect_b32 s2, -1, 0
; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-NEXT: s_or_b32 s0, s1, s0
-; GCN-NEXT: s_and_b32 s0, s6, s0
+; GCN-NEXT: s_or_b32 s1, s2, s1
+; GCN-NEXT: s_and_b32 s0, s0, s1
; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GCN-NEXT: ; %bb.8: ; %bb14
@@ -1207,80 +1193,76 @@ define amdgpu_kernel void @long_branch_hang(ptr addrspace(1) nocapture %arg, i32
; GCN-NEXT: v_nop_e64
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: s_sleep 0
-; GCN-NEXT: s_mov_b32 s0, s8
+; GCN-NEXT: s_mov_b32 s1, s3
; GCN-NEXT: s_sleep 0
; GCN-NEXT: s_cbranch_execz .LBB10_5
; GCN-NEXT: ; %bb.16: ; %bb13
-; GCN-NEXT: s_get_pc_i64 s[8:9]
+; GCN-NEXT: s_get_pc_i64 s[2:3]
; GCN-NEXT: .Lpost_getpc17:
-; GCN-NEXT: s_add_co_u32 s8, s8, (.LBB10_2-.Lpost_getpc17)&4294967295
-; GCN-NEXT: s_add_co_ci_u32 s9, s9, (.LBB10_2-.Lpost_getpc17)>>32
-; GCN-NEXT: s_set_pc_i64 s[8:9]
+; GCN-NEXT: s_add_co_u32 s2, s2, (.LBB10_2-.Lpost_getpc17)&4294967295
+; GCN-NEXT: s_add_co_ci_u32 s3, s3, (.LBB10_2-.Lpost_getpc17)>>32
+; GCN-NEXT: s_set_pc_i64 s[2:3]
; GCN-NEXT: .LBB10_5: ; %bb9
-; GCN-NEXT: s_cmp_lt_i32 s3, 11
-; GCN-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-NEXT: s_cmp_ge_i32 s2, s3
-; GCN-NEXT: s_cselect_b32 s7, -1, 0
+; GCN-NEXT: s_cmp_lt_i32 s13, 11
+; GCN-NEXT: s_cselect_b32 s1, -1, 0
+; GCN-NEXT: s_cmp_ge_i32 s12, s13
+; GCN-NEXT: s_cselect_b32 s2, -1, 0
; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-NEXT: s_and_b32 s0, s7, s0
-; GCN-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
+; GCN-NEXT: s_and_b32 s1, s2, s1
+; GCN-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
; GCN-NEXT: s_cbranch_vccnz .LBB10_6
; GCN-NEXT: ; %bb.18: ; %bb9
-; GCN-NEXT: s_get_pc_i64 s[8:9]
+; GCN-NEXT: s_get_pc_i64 s[2:3]
; GCN-NEXT: .Lpost_getpc18:
-; GCN-NEXT: s_add_co_u32 s8, s8, (.LBB10_3-.Lpost_getpc18)&4294967295
-; GCN-NEXT: s_add_co_ci_u32 s9, s9, (.LBB10_3-.Lpost_getpc18)>>32
-; GCN-NEXT: s_set_pc_i64 s[8:9]
+; GCN-NEXT: s_add_co_u32 s2, s2, (.LBB10_3-.Lpost_getpc18)&4294967295
+; GCN-NEXT: s_add_co_ci_u32 s3, s3, (.LBB10_3-.Lpost_getpc18)>>32
+; GCN-NEXT: s_set_pc_i64 s[2:3]
; GCN-NEXT: .LBB10_6:
; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: .LBB10_7: ; %bb19
-; GCN-NEXT: s_clause 0x1
-; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x3c nv
-; GCN-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN-NEXT: s_lshl_b64 s[0:1], s[14:15], 2
; GCN-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GCN-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
+; GCN-NEXT: s_add_nc_u64 s[0:1], s[8:9], s[0:1]
; GCN-NEXT: global_store_b32 v1, v0, s[0:1]
; GCN-NEXT: s_endpgm
;
; GCN-ADD-PC64-LABEL: long_branch_hang:
; GCN-ADD-PC64: ; %bb.0: ; %bb
; GCN-ADD-PC64-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GCN-ADD-PC64-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c nv
-; GCN-ADD-PC64-NEXT: s_mov_b32 s7, -1
+; GCN-ADD-PC64-NEXT: s_load_b256 s[8:15], s[4:5], 0x24 nv
+; GCN-ADD-PC64-NEXT: s_mov_b32 s1, 0
+; GCN-ADD-PC64-NEXT: s_mov_b32 s2, -1
; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
-; GCN-ADD-PC64-NEXT: s_cmp_eq_u32 s0, 0
-; GCN-ADD-PC64-NEXT: s_cselect_b32 s6, -1, 0
-; GCN-ADD-PC64-NEXT: s_cmp_lg_u32 s0, 0
-; GCN-ADD-PC64-NEXT: s_mov_b32 s0, 0
-; GCN-ADD-PC64-NEXT: s_cselect_b32 s8, -1, 0
-; GCN-ADD-PC64-NEXT: s_cmp_lt_i32 s3, 6
+; GCN-ADD-PC64-NEXT: s_cmp_eq_u32 s10, 0
+; GCN-ADD-PC64-NEXT: s_cselect_b32 s0, -1, 0
+; GCN-ADD-PC64-NEXT: s_cmp_lg_u32 s10, 0
+; GCN-ADD-PC64-NEXT: s_cselect_b32 s3, -1, 0
+; GCN-ADD-PC64-NEXT: s_cmp_lt_i32 s13, 6
; GCN-ADD-PC64-NEXT: s_cbranch_scc0 .LBB10_1
; GCN-ADD-PC64-NEXT: ; %bb.10: ; %bb
; GCN-ADD-PC64-NEXT: s_add_pc_i64 .LBB10_4-.Lpost_addpc14
; GCN-ADD-PC64-NEXT: .Lpost_addpc14:
; GCN-ADD-PC64-NEXT: .LBB10_1: ; %Flow
-; GCN-ADD-PC64-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s7
+; GCN-ADD-PC64-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
; GCN-ADD-PC64-NEXT: s_cbranch_vccnz .LBB10_2
; GCN-ADD-PC64-NEXT: ; %bb.12: ; %Flow
; GCN-ADD-PC64-NEXT: s_add_pc_i64 .LBB10_5-.Lpost_addpc15
; GCN-ADD-PC64-NEXT: .Lpost_addpc15:
-; GCN-ADD-PC64-NEXT: .LBB10_2: ; %Flow5
-; GCN-ADD-PC64-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
+; GCN-ADD-PC64-NEXT: .LBB10_2: ; %Flow7
+; GCN-ADD-PC64-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
; GCN-ADD-PC64-NEXT: s_cbranch_vccz .LBB10_3
-; GCN-ADD-PC64-NEXT: ; %bb.14: ; %Flow5
+; GCN-ADD-PC64-NEXT: ; %bb.14: ; %Flow7
; GCN-ADD-PC64-NEXT: s_add_pc_i64 .LBB10_6-.Lpost_addpc16
; GCN-ADD-PC64-NEXT: .Lpost_addpc16:
; GCN-ADD-PC64-NEXT: .LBB10_3: ; %bb14
-; GCN-ADD-PC64-NEXT: s_cmp_lt_i32 s1, 9
-; GCN-ADD-PC64-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-ADD-PC64-NEXT: s_cmp_lt_i32 s2, s3
+; GCN-ADD-PC64-NEXT: s_cmp_lt_i32 s11, 9
; GCN-ADD-PC64-NEXT: s_cselect_b32 s1, -1, 0
+; GCN-ADD-PC64-NEXT: s_cmp_lt_i32 s12, s13
+; GCN-ADD-PC64-NEXT: s_cselect_b32 s2, -1, 0
; GCN-ADD-PC64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-ADD-PC64-NEXT: s_or_b32 s0, s1, s0
-; GCN-ADD-PC64-NEXT: s_and_b32 s0, s6, s0
+; GCN-ADD-PC64-NEXT: s_or_b32 s1, s2, s1
+; GCN-ADD-PC64-NEXT: s_and_b32 s0, s0, s1
; GCN-ADD-PC64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GCN-ADD-PC64-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
; GCN-ADD-PC64-NEXT: ; %bb.8: ; %bb14
@@ -1292,20 +1274,20 @@ define amdgpu_kernel void @long_branch_hang(ptr addrspace(1) nocapture %arg, i32
; GCN-ADD-PC64-NEXT: v_nop_e64
; GCN-ADD-PC64-NEXT: ;;#ASMEND
; GCN-ADD-PC64-NEXT: s_sleep 0
-; GCN-ADD-PC64-NEXT: s_mov_b32 s0, s8
+; GCN-ADD-PC64-NEXT: s_mov_b32 s1, s3
; GCN-ADD-PC64-NEXT: s_sleep 0
; GCN-ADD-PC64-NEXT: s_cbranch_execz .LBB10_5
; GCN-ADD-PC64-NEXT: ; %bb.16: ; %bb13
; GCN-ADD-PC64-NEXT: s_add_pc_i64 .LBB10_2-.Lpost_addpc17
; GCN-ADD-PC64-NEXT: .Lpost_addpc17:
; GCN-ADD-PC64-NEXT: .LBB10_5: ; %bb9
-; GCN-ADD-PC64-NEXT: s_cmp_lt_i32 s3, 11
-; GCN-ADD-PC64-NEXT: s_cselect_b32 s0, -1, 0
-; GCN-ADD-PC64-NEXT: s_cmp_ge_i32 s2, s3
-; GCN-ADD-PC64-NEXT: s_cselect_b32 s7, -1, 0
+; GCN-ADD-PC64-NEXT: s_cmp_lt_i32 s13, 11
+; GCN-ADD-PC64-NEXT: s_cselect_b32 s1, -1, 0
+; GCN-ADD-PC64-NEXT: s_cmp_ge_i32 s12, s13
+; GCN-ADD-PC64-NEXT: s_cselect_b32 s2, -1, 0
; GCN-ADD-PC64-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GCN-ADD-PC64-NEXT: s_and_b32 s0, s7, s0
-; GCN-ADD-PC64-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
+; GCN-ADD-PC64-NEXT: s_and_b32 s1, s2, s1
+; GCN-ADD-PC64-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s1
; GCN-ADD-PC64-NEXT: s_cbranch_vccnz .LBB10_6
; GCN-ADD-PC64-NEXT: ; %bb.18: ; %bb9
; GCN-ADD-PC64-NEXT: s_add_pc_i64 .LBB10_3-.Lpost_addpc18
@@ -1313,14 +1295,10 @@ define amdgpu_kernel void @long_branch_hang(ptr addrspace(1) nocapture %arg, i32
; GCN-ADD-PC64-NEXT: .LBB10_6:
; GCN-ADD-PC64-NEXT: ; implicit-def: $vgpr0
; GCN-ADD-PC64-NEXT: .LBB10_7: ; %bb19
-; GCN-ADD-PC64-NEXT: s_clause 0x1
-; GCN-ADD-PC64-NEXT: s_load_b64 s[0:1], s[4:5], 0x3c nv
-; GCN-ADD-PC64-NEXT: s_load_b64 s[2:3], s[4:5], 0x24 nv
; GCN-ADD-PC64-NEXT: v_mov_b32_e32 v1, 0
-; GCN-ADD-PC64-NEXT: s_wait_kmcnt 0x0
-; GCN-ADD-PC64-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN-ADD-PC64-NEXT: s_lshl_b64 s[0:1], s[14:15], 2
; GCN-ADD-PC64-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GCN-ADD-PC64-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
+; GCN-ADD-PC64-NEXT: s_add_nc_u64 s[0:1], s[8:9], s[0:1]
; GCN-ADD-PC64-NEXT: global_store_b32 v1, v0, s[0:1]
; GCN-ADD-PC64-NEXT: s_endpgm
; GCN-ENABLE-ADD-PC64-LABEL: long_branch_hang:
diff --git a/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll b/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
index 155e54baf15a1..d0aec6a581a4b 100644
--- a/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
+++ b/llvm/test/CodeGen/AMDGPU/branch-relaxation.ll
@@ -24,9 +24,9 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addrspace(1) %arg, i32 %cnd) #0 {
; GCN-LABEL: uniform_conditional_max_short_forward_branch:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dword s0, s[4:5], 0xb
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s0, 0
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
; GCN-NEXT: s_cbranch_scc1 .LBB0_2
; GCN-NEXT: ; %bb.1: ; %bb2
; GCN-NEXT: ;;#ASMSTART
@@ -36,29 +36,29 @@ define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addr
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: s_sleep 0
; GCN-NEXT: .LBB0_2: ; %bb3
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
;
; GFX11-LABEL: uniform_conditional_max_short_forward_branch:
; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_cmp_eq_u32 s0, 0
+; GFX11-NEXT: s_cmp_eq_u32 s2, 0
; GFX11-NEXT: s_cbranch_scc0 .LBB0_1
; GFX11-NEXT: ; %bb.3: ; %bb
-; GFX11-NEXT: s_getpc_b64 s[2:3]
+; GFX11-NEXT: s_getpc_b64 s[4:5]
; GFX11-NEXT: .Lpost_getpc0:
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_add_u32 s2, s2, (.LBB0_2-.Lpost_getpc0)&4294967295
-; GFX11-NEXT: s_addc_u32 s3, s3, (.LBB0_2-.Lpost_getpc0)>>32
+; GFX11-NEXT: s_add_u32 s4, s4, (.LBB0_2-.Lpost_getpc0)&4294967295
+; GFX11-NEXT: s_addc_u32 s5, s5, (.LBB0_2-.Lpost_getpc0)>>32
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_setpc_b64 s[2:3]
+; GFX11-NEXT: s_setpc_b64 s[4:5]
; GFX11-NEXT: .LBB0_1: ; %bb2
; GFX11-NEXT: ;;#ASMSTART
; GFX11-NEXT: v_nop_e64
@@ -67,28 +67,26 @@ define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addr
; GFX11-NEXT: ;;#ASMEND
; GFX11-NEXT: s_sleep 0
; GFX11-NEXT: .LBB0_2: ; %bb3
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: v_mov_b32_e32 v1, s0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_store_b32 v0, v1, s[2:3] dlc
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: uniform_conditional_max_short_forward_branch:
; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_cmp_eq_u32 s0, 0
+; GFX12-NEXT: s_cmp_eq_u32 s2, 0
; GFX12-NEXT: s_cbranch_scc0 .LBB0_1
; GFX12-NEXT: ; %bb.3: ; %bb
-; GFX12-NEXT: s_getpc_b64 s[2:3]
+; GFX12-NEXT: s_getpc_b64 s[4:5]
; GFX12-NEXT: .Lpost_getpc0:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s2, s2, (.LBB0_2-.Lpost_getpc0)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s3, s3, (.LBB0_2-.Lpost_getpc0)>>32
+; GFX12-NEXT: s_add_co_u32 s4, s4, (.LBB0_2-.Lpost_getpc0)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s5, s5, (.LBB0_2-.Lpost_getpc0)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[2:3]
+; GFX12-NEXT: s_setpc_b64 s[4:5]
; GFX12-NEXT: .LBB0_1: ; %bb2
; GFX12-NEXT: ;;#ASMSTART
; GFX12-NEXT: v_nop_e64
@@ -97,10 +95,8 @@ define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addr
; GFX12-NEXT: ;;#ASMEND
; GFX12-NEXT: s_sleep 0
; GFX12-NEXT: .LBB0_2: ; %bb3
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_endpgm
bb:
@@ -124,16 +120,16 @@ bb3:
define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrspace(1) %arg, i32 %cnd) #0 {
; GCN-LABEL: uniform_conditional_min_long_forward_branch:
; GCN: ; %bb.0: ; %bb0
-; GCN-NEXT: s_load_dword s0, s[4:5], 0xb
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s0, 0
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
; GCN-NEXT: s_cbranch_scc0 .LBB1_1
; GCN-NEXT: ; %bb.3: ; %bb0
-; GCN-NEXT: s_getpc_b64 s[2:3]
+; GCN-NEXT: s_getpc_b64 s[4:5]
; GCN-NEXT: .Lpost_getpc0:
-; GCN-NEXT: s_add_u32 s2, s2, (.LBB1_2-.Lpost_getpc0)&4294967295
-; GCN-NEXT: s_addc_u32 s3, s3, (.LBB1_2-.Lpost_getpc0)>>32
-; GCN-NEXT: s_setpc_b64 s[2:3]
+; GCN-NEXT: s_add_u32 s4, s4, (.LBB1_2-.Lpost_getpc0)&4294967295
+; GCN-NEXT: s_addc_u32 s5, s5, (.LBB1_2-.Lpost_getpc0)>>32
+; GCN-NEXT: s_setpc_b64 s[4:5]
; GCN-NEXT: .LBB1_1: ; %bb2
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: v_nop_e64
@@ -142,29 +138,29 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrs
; GCN-NEXT: v_nop_e64
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: .LBB1_2: ; %bb3
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
;
; GFX11-LABEL: uniform_conditional_min_long_forward_branch:
; GFX11: ; %bb.0: ; %bb0
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_cmp_eq_u32 s0, 0
+; GFX11-NEXT: s_cmp_eq_u32 s2, 0
; GFX11-NEXT: s_cbranch_scc0 .LBB1_1
; GFX11-NEXT: ; %bb.3: ; %bb0
-; GFX11-NEXT: s_getpc_b64 s[2:3]
+; GFX11-NEXT: s_getpc_b64 s[4:5]
; GFX11-NEXT: .Lpost_getpc1:
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_add_u32 s2, s2, (.LBB1_2-.Lpost_getpc1)&4294967295
-; GFX11-NEXT: s_addc_u32 s3, s3, (.LBB1_2-.Lpost_getpc1)>>32
+; GFX11-NEXT: s_add_u32 s4, s4, (.LBB1_2-.Lpost_getpc1)&4294967295
+; GFX11-NEXT: s_addc_u32 s5, s5, (.LBB1_2-.Lpost_getpc1)>>32
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_setpc_b64 s[2:3]
+; GFX11-NEXT: s_setpc_b64 s[4:5]
; GFX11-NEXT: .LBB1_1: ; %bb2
; GFX11-NEXT: ;;#ASMSTART
; GFX11-NEXT: v_nop_e64
@@ -173,28 +169,26 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrs
; GFX11-NEXT: v_nop_e64
; GFX11-NEXT: ;;#ASMEND
; GFX11-NEXT: .LBB1_2: ; %bb3
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: v_mov_b32_e32 v1, s0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_store_b32 v0, v1, s[2:3] dlc
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: uniform_conditional_min_long_forward_branch:
; GFX12: ; %bb.0: ; %bb0
-; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_cmp_eq_u32 s0, 0
+; GFX12-NEXT: s_cmp_eq_u32 s2, 0
; GFX12-NEXT: s_cbranch_scc0 .LBB1_1
; GFX12-NEXT: ; %bb.3: ; %bb0
-; GFX12-NEXT: s_getpc_b64 s[2:3]
+; GFX12-NEXT: s_getpc_b64 s[4:5]
; GFX12-NEXT: .Lpost_getpc1:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s2, s2, (.LBB1_2-.Lpost_getpc1)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s3, s3, (.LBB1_2-.Lpost_getpc1)>>32
+; GFX12-NEXT: s_add_co_u32 s4, s4, (.LBB1_2-.Lpost_getpc1)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s5, s5, (.LBB1_2-.Lpost_getpc1)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[2:3]
+; GFX12-NEXT: s_setpc_b64 s[4:5]
; GFX12-NEXT: .LBB1_1: ; %bb2
; GFX12-NEXT: ;;#ASMSTART
; GFX12-NEXT: v_nop_e64
@@ -203,10 +197,8 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrs
; GFX12-NEXT: v_nop_e64
; GFX12-NEXT: ;;#ASMEND
; GFX12-NEXT: .LBB1_2: ; %bb3
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_endpgm
bb0:
@@ -230,17 +222,17 @@ bb3:
define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr addrspace(1) %arg, float %cnd) #0 {
; GCN-LABEL: uniform_conditional_min_long_forward_vcnd_branch:
; GCN: ; %bb.0: ; %bb0
-; GCN-NEXT: s_load_dword s0, s[4:5], 0xb
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_cmp_eq_f32_e64 s[2:3], s0, 0
-; GCN-NEXT: s_and_b64 vcc, exec, s[2:3]
+; GCN-NEXT: v_cmp_eq_f32_e64 s[4:5], s2, 0
+; GCN-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN-NEXT: s_cbranch_vccz .LBB2_1
; GCN-NEXT: ; %bb.3: ; %bb0
-; GCN-NEXT: s_getpc_b64 s[2:3]
+; GCN-NEXT: s_getpc_b64 s[4:5]
; GCN-NEXT: .Lpost_getpc1:
-; GCN-NEXT: s_add_u32 s2, s2, (.LBB2_2-.Lpost_getpc1)&4294967295
-; GCN-NEXT: s_addc_u32 s3, s3, (.LBB2_2-.Lpost_getpc1)>>32
-; GCN-NEXT: s_setpc_b64 s[2:3]
+; GCN-NEXT: s_add_u32 s4, s4, (.LBB2_2-.Lpost_getpc1)&4294967295
+; GCN-NEXT: s_addc_u32 s5, s5, (.LBB2_2-.Lpost_getpc1)>>32
+; GCN-NEXT: s_setpc_b64 s[4:5]
; GCN-NEXT: .LBB2_1: ; %bb2
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: ; 32 bytes
@@ -250,30 +242,30 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr
; GCN-NEXT: v_nop_e64
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: .LBB2_2: ; %bb3
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
;
; GFX11-LABEL: uniform_conditional_min_long_forward_vcnd_branch:
; GFX11: ; %bb.0: ; %bb0
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_eq_f32_e64 s[2:3], s0, 0
-; GFX11-NEXT: s_and_b64 vcc, exec, s[2:3]
+; GFX11-NEXT: v_cmp_eq_f32_e64 s[4:5], s2, 0
+; GFX11-NEXT: s_and_b64 vcc, exec, s[4:5]
; GFX11-NEXT: s_cbranch_vccz .LBB2_1
; GFX11-NEXT: ; %bb.3: ; %bb0
-; GFX11-NEXT: s_getpc_b64 s[2:3]
+; GFX11-NEXT: s_getpc_b64 s[4:5]
; GFX11-NEXT: .Lpost_getpc2:
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_add_u32 s2, s2, (.LBB2_2-.Lpost_getpc2)&4294967295
-; GFX11-NEXT: s_addc_u32 s3, s3, (.LBB2_2-.Lpost_getpc2)>>32
+; GFX11-NEXT: s_add_u32 s4, s4, (.LBB2_2-.Lpost_getpc2)&4294967295
+; GFX11-NEXT: s_addc_u32 s5, s5, (.LBB2_2-.Lpost_getpc2)>>32
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_setpc_b64 s[2:3]
+; GFX11-NEXT: s_setpc_b64 s[4:5]
; GFX11-NEXT: .LBB2_1: ; %bb2
; GFX11-NEXT: ;;#ASMSTART
; GFX11-NEXT: ; 32 bytes
@@ -283,28 +275,26 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr
; GFX11-NEXT: v_nop_e64
; GFX11-NEXT: ;;#ASMEND
; GFX11-NEXT: .LBB2_2: ; %bb3
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: v_mov_b32_e32 v1, s0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_store_b32 v0, v1, s[2:3] dlc
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: uniform_conditional_min_long_forward_vcnd_branch:
; GFX12: ; %bb.0: ; %bb0
-; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_cmp_eq_f32 s0, 0
+; GFX12-NEXT: s_cmp_eq_f32 s2, 0
; GFX12-NEXT: s_cbranch_scc0 .LBB2_1
; GFX12-NEXT: ; %bb.3: ; %bb0
-; GFX12-NEXT: s_getpc_b64 s[2:3]
+; GFX12-NEXT: s_getpc_b64 s[4:5]
; GFX12-NEXT: .Lpost_getpc2:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s2, s2, (.LBB2_2-.Lpost_getpc2)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s3, s3, (.LBB2_2-.Lpost_getpc2)>>32
+; GFX12-NEXT: s_add_co_u32 s4, s4, (.LBB2_2-.Lpost_getpc2)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s5, s5, (.LBB2_2-.Lpost_getpc2)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[2:3]
+; GFX12-NEXT: s_setpc_b64 s[4:5]
; GFX12-NEXT: .LBB2_1: ; %bb2
; GFX12-NEXT: ;;#ASMSTART
; GFX12-NEXT: ; 32 bytes
@@ -314,10 +304,8 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr
; GFX12-NEXT: v_nop_e64
; GFX12-NEXT: ;;#ASMEND
; GFX12-NEXT: .LBB2_2: ; %bb3
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_store_b32 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_endpgm
bb0:
@@ -570,19 +558,19 @@ bb3:
define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr addrspace(1) %arg, i32 %arg1) {
; GCN-LABEL: uniform_unconditional_min_long_forward_branch:
; GCN: ; %bb.0: ; %bb0
-; GCN-NEXT: s_load_dword s0, s[4:5], 0xb
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s0, 0
-; GCN-NEXT: s_mov_b64 s[0:1], -1
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-NEXT: s_mov_b64 s[2:3], -1
; GCN-NEXT: s_cbranch_scc0 .LBB5_1
; GCN-NEXT: ; %bb.7: ; %bb0
-; GCN-NEXT: s_getpc_b64 s[0:1]
+; GCN-NEXT: s_getpc_b64 s[2:3]
; GCN-NEXT: .Lpost_getpc5:
-; GCN-NEXT: s_add_u32 s0, s0, (.LBB5_4-.Lpost_getpc5)&4294967295
-; GCN-NEXT: s_addc_u32 s1, s1, (.LBB5_4-.Lpost_getpc5)>>32
-; GCN-NEXT: s_setpc_b64 s[0:1]
+; GCN-NEXT: s_add_u32 s2, s2, (.LBB5_4-.Lpost_getpc5)&4294967295
+; GCN-NEXT: s_addc_u32 s3, s3, (.LBB5_4-.Lpost_getpc5)>>32
+; GCN-NEXT: s_setpc_b64 s[2:3]
; GCN-NEXT: .LBB5_1: ; %Flow
-; GCN-NEXT: s_andn2_b64 vcc, exec, s[0:1]
+; GCN-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; GCN-NEXT: s_cbranch_vccnz .LBB5_3
; GCN-NEXT: .LBB5_2: ; %bb2
; GCN-NEXT: s_mov_b32 s3, 0xf000
@@ -591,12 +579,10 @@ define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr add
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: .LBB5_3: ; %bb4
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_mov_b32 s2, -1
; GCN-NEXT: s_waitcnt expcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, 63
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
@@ -609,37 +595,35 @@ define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr add
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: s_cbranch_execnz .LBB5_5
; GCN-NEXT: ; %bb.9: ; %bb3
-; GCN-NEXT: s_getpc_b64 s[0:1]
+; GCN-NEXT: s_getpc_b64 s[2:3]
; GCN-NEXT: .Lpost_getpc6:
-; GCN-NEXT: s_add_u32 s0, s0, (.LBB5_2-.Lpost_getpc6)&4294967295
-; GCN-NEXT: s_addc_u32 s1, s1, (.LBB5_2-.Lpost_getpc6)>>32
-; GCN-NEXT: s_setpc_b64 s[0:1]
+; GCN-NEXT: s_add_u32 s2, s2, (.LBB5_2-.Lpost_getpc6)&4294967295
+; GCN-NEXT: s_addc_u32 s3, s3, (.LBB5_2-.Lpost_getpc6)>>32
+; GCN-NEXT: s_setpc_b64 s[2:3]
; GCN-NEXT: .LBB5_5: ; %bb3
-; GCN-NEXT: s_getpc_b64 s[0:1]
+; GCN-NEXT: s_getpc_b64 s[2:3]
; GCN-NEXT: .Lpost_getpc4:
-; GCN-NEXT: s_add_u32 s0, s0, (.LBB5_3-.Lpost_getpc4)&4294967295
-; GCN-NEXT: s_addc_u32 s1, s1, (.LBB5_3-.Lpost_getpc4)>>32
-; GCN-NEXT: s_setpc_b64 s[0:1]
+; GCN-NEXT: s_add_u32 s2, s2, (.LBB5_3-.Lpost_getpc4)&4294967295
+; GCN-NEXT: s_addc_u32 s3, s3, (.LBB5_3-.Lpost_getpc4)>>32
+; GCN-NEXT: s_setpc_b64 s[2:3]
;
; GFX11-LABEL: uniform_unconditional_min_long_forward_branch:
; GFX11: ; %bb.0: ; %bb0
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_cmp_eq_u32 s0, 0
-; GFX11-NEXT: s_mov_b64 s[0:1], -1
+; GFX11-NEXT: s_cmp_eq_u32 s2, 0
+; GFX11-NEXT: s_mov_b64 s[2:3], -1
; GFX11-NEXT: s_cbranch_scc1 .LBB5_4
; GFX11-NEXT: ; %bb.1: ; %Flow
-; GFX11-NEXT: s_and_not1_b64 vcc, exec, s[0:1]
+; GFX11-NEXT: s_and_not1_b64 vcc, exec, s[2:3]
; GFX11-NEXT: s_cbranch_vccnz .LBB5_3
; GFX11-NEXT: .LBB5_2: ; %bb2
; GFX11-NEXT: v_mov_b32_e32 v0, 17
; GFX11-NEXT: global_store_b32 v[0:1], v0, off dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: .LBB5_3: ; %bb4
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: v_mov_b32_e32 v1, 63
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_endpgm
@@ -652,48 +636,38 @@ define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr add
; GFX11-NEXT: ;;#ASMEND
; GFX11-NEXT: s_cbranch_execnz .LBB5_5
; GFX11-NEXT: ; %bb.7: ; %bb3
-; GFX11-NEXT: s_getpc_b64 s[0:1]
+; GFX11-NEXT: s_getpc_b64 s[2:3]
; GFX11-NEXT: .Lpost_getpc6:
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_add_u32 s0, s0, (.LBB5_2-.Lpost_getpc6)&4294967295
-; GFX11-NEXT: s_addc_u32 s1, s1, (.LBB5_2-.Lpost_getpc6)>>32
+; GFX11-NEXT: s_add_u32 s2, s2, (.LBB5_2-.Lpost_getpc6)&4294967295
+; GFX11-NEXT: s_addc_u32 s3, s3, (.LBB5_2-.Lpost_getpc6)>>32
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_setpc_b64 s[0:1]
+; GFX11-NEXT: s_setpc_b64 s[2:3]
; GFX11-NEXT: .LBB5_5: ; %bb3
-; GFX11-NEXT: s_getpc_b64 s[0:1]
+; GFX11-NEXT: s_getpc_b64 s[2:3]
; GFX11-NEXT: .Lpost_getpc5:
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_add_u32 s0, s0, (.LBB5_3-.Lpost_getpc5)&4294967295
-; GFX11-NEXT: s_addc_u32 s1, s1, (.LBB5_3-.Lpost_getpc5)>>32
+; GFX11-NEXT: s_add_u32 s2, s2, (.LBB5_3-.Lpost_getpc5)&4294967295
+; GFX11-NEXT: s_addc_u32 s3, s3, (.LBB5_3-.Lpost_getpc5)>>32
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_setpc_b64 s[0:1]
+; GFX11-NEXT: s_setpc_b64 s[2:3]
;
; GFX12-LABEL: uniform_unconditional_min_long_forward_branch:
; GFX12: ; %bb.0: ; %bb0
-; GFX12-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_cmp_eq_u32 s0, 0
-; GFX12-NEXT: s_mov_b32 s0, -1
-; GFX12-NEXT: s_cbranch_scc0 .LBB5_1
-; GFX12-NEXT: ; %bb.7: ; %bb0
-; GFX12-NEXT: s_getpc_b64 s[0:1]
-; GFX12-NEXT: .Lpost_getpc6:
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB5_4-.Lpost_getpc6)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB5_4-.Lpost_getpc6)>>32
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[0:1]
-; GFX12-NEXT: .LBB5_1: ; %Flow
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
+; GFX12-NEXT: s_cmp_eq_u32 s2, 0
+; GFX12-NEXT: s_mov_b32 s2, -1
+; GFX12-NEXT: s_cbranch_scc1 .LBB5_4
+; GFX12-NEXT: ; %bb.1: ; %Flow
+; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
; GFX12-NEXT: s_cbranch_vccnz .LBB5_3
; GFX12-NEXT: .LBB5_2: ; %bb2
; GFX12-NEXT: v_mov_b32_e32 v0, 17
; GFX12-NEXT: global_store_b32 v[0:1], v0, off scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: .LBB5_3: ; %bb4
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, 63
-; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_store_b32 v0, v1, s[0:1] scope:SCOPE_SYS
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_endpgm
@@ -705,22 +679,22 @@ define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr add
; GFX12-NEXT: v_nop_e64
; GFX12-NEXT: ;;#ASMEND
; GFX12-NEXT: s_cbranch_execnz .LBB5_5
-; GFX12-NEXT: ; %bb.9: ; %bb3
-; GFX12-NEXT: s_getpc_b64 s[0:1]
-; GFX12-NEXT: .Lpost_getpc7:
+; GFX12-NEXT: ; %bb.7: ; %bb3
+; GFX12-NEXT: s_getpc_b64 s[2:3]
+; GFX12-NEXT: .Lpost_getpc6:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB5_2-.Lpost_getpc7)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB5_2-.Lpost_getpc7)>>32
+; GFX12-NEXT: s_add_co_u32 s2, s2, (.LBB5_2-.Lpost_getpc6)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s3, s3, (.LBB5_2-.Lpost_getpc6)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[0:1]
+; GFX12-NEXT: s_setpc_b64 s[2:3]
; GFX12-NEXT: .LBB5_5: ; %bb3
-; GFX12-NEXT: s_getpc_b64 s[0:1]
+; GFX12-NEXT: s_getpc_b64 s[2:3]
; GFX12-NEXT: .Lpost_getpc5:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB5_3-.Lpost_getpc5)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB5_3-.Lpost_getpc5)>>32
+; GFX12-NEXT: s_add_co_u32 s2, s2, (.LBB5_3-.Lpost_getpc5)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s3, s3, (.LBB5_3-.Lpost_getpc5)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[0:1]
+; GFX12-NEXT: s_setpc_b64 s[2:3]
bb0:
%tmp = icmp ne i32 %arg1, 0
br i1 %tmp, label %bb2, label %bb3
@@ -807,10 +781,10 @@ define amdgpu_kernel void @uniform_unconditional_min_long_backward_branch(ptr ad
; GFX12-NEXT: ; %bb.3: ; %loop
; GFX12-NEXT: ; in Loop: Header=BB6_1 Depth=1
; GFX12-NEXT: s_getpc_b64 s[0:1]
-; GFX12-NEXT: .Lpost_getpc8:
+; GFX12-NEXT: .Lpost_getpc7:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB6_1-.Lpost_getpc8)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB6_1-.Lpost_getpc8)>>32
+; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB6_1-.Lpost_getpc7)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB6_1-.Lpost_getpc7)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[0:1]
; GFX12-NEXT: .LBB6_2: ; %DummyReturnBlock
@@ -932,10 +906,10 @@ define amdgpu_kernel void @expand_requires_expand(i32 %cond0) #0 {
; GFX12-NEXT: s_cbranch_vccz .LBB7_3
; GFX12-NEXT: ; %bb.5: ; %Flow
; GFX12-NEXT: s_getpc_b64 s[0:1]
-; GFX12-NEXT: .Lpost_getpc9:
+; GFX12-NEXT: .Lpost_getpc8:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB7_4-.Lpost_getpc9)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB7_4-.Lpost_getpc9)>>32
+; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB7_4-.Lpost_getpc8)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB7_4-.Lpost_getpc8)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[0:1]
; GFX12-NEXT: .LBB7_3: ; %bb2
@@ -987,7 +961,7 @@ define amdgpu_kernel void @uniform_inside_divergent(ptr addrspace(1) %out, i32 %
; GCN-LABEL: uniform_inside_divergent:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0
-; GCN-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GCN-NEXT: s_and_saveexec_b64 s[8:9], vcc
; GCN-NEXT: s_cbranch_execnz .LBB8_1
; GCN-NEXT: ; %bb.4: ; %entry
; GCN-NEXT: s_getpc_b64 s[0:1]
@@ -996,45 +970,46 @@ define amdgpu_kernel void @uniform_inside_divergent(ptr addrspace(1) %out, i32 %
; GCN-NEXT: s_addc_u32 s1, s1, (.LBB8_3-.Lpost_getpc9)>>32
; GCN-NEXT: s_setpc_b64 s[0:1]
; GCN-NEXT: .LBB8_1: ; %if
-; GCN-NEXT: s_load_dword s8, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_lg_u32 s8, 0
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_cmp_lg_u32 s2, 0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_cbranch_scc1 .LBB8_3
; GCN-NEXT: ; %bb.2: ; %if_uniform
+; GCN-NEXT: s_mov_b32 s2, s6
+; GCN-NEXT: s_mov_b32 s3, s7
; GCN-NEXT: s_waitcnt expcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, 1
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: .LBB8_3: ; %endif
-; GCN-NEXT: s_or_b64 exec, exec, s[6:7]
+; GCN-NEXT: s_or_b64 exec, exec, s[8:9]
; GCN-NEXT: s_sleep 5
; GCN-NEXT: s_endpgm
;
; GFX11-LABEL: uniform_inside_divergent:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: s_mov_b64 s[0:1], exec
+; GFX11-NEXT: s_mov_b64 s[6:7], exec
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_cmpx_gt_u32_e32 16, v0
; GFX11-NEXT: s_cbranch_execz .LBB8_3
; GFX11-NEXT: ; %bb.1: ; %if
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s6, 0
-; GFX11-NEXT: global_store_b32 v0, v0, s[2:3]
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-NEXT: global_store_b32 v0, v0, s[0:1]
; GFX11-NEXT: s_cbranch_scc1 .LBB8_3
; GFX11-NEXT: ; %bb.2: ; %if_uniform
; GFX11-NEXT: v_mov_b32_e32 v1, 1
-; GFX11-NEXT: global_store_b32 v0, v1, s[2:3]
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: .LBB8_3: ; %endif
-; GFX11-NEXT: s_or_b64 exec, exec, s[0:1]
+; GFX11-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX11-NEXT: s_sleep 5
; GFX11-NEXT: s_endpgm
;
@@ -1203,10 +1178,10 @@ define amdgpu_kernel void @analyze_mask_branch() #0 {
; GFX12-NEXT: s_cbranch_execnz .LBB9_3
; GFX12-NEXT: ; %bb.6: ; %Flow1
; GFX12-NEXT: s_getpc_b64 s[0:1]
-; GFX12-NEXT: .Lpost_getpc10:
+; GFX12-NEXT: .Lpost_getpc9:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB9_5-.Lpost_getpc10)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB9_5-.Lpost_getpc10)>>32
+; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB9_5-.Lpost_getpc9)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB9_5-.Lpost_getpc9)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[0:1]
; GFX12-NEXT: .LBB9_3: ; %loop.preheader
@@ -1227,10 +1202,10 @@ define amdgpu_kernel void @analyze_mask_branch() #0 {
; GFX12-NEXT: ; %bb.8: ; %loop
; GFX12-NEXT: ; in Loop: Header=BB9_4 Depth=1
; GFX12-NEXT: s_getpc_b64 s[0:1]
-; GFX12-NEXT: .Lpost_getpc11:
+; GFX12-NEXT: .Lpost_getpc10:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB9_4-.Lpost_getpc11)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB9_4-.Lpost_getpc11)>>32
+; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB9_4-.Lpost_getpc10)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB9_4-.Lpost_getpc10)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_setpc_b64 s[0:1]
; GFX12-NEXT: .LBB9_5: ; %UnifiedReturnBlock
@@ -1264,20 +1239,20 @@ ret:
define amdgpu_kernel void @long_branch_hang(ptr addrspace(1) nocapture %arg, i32 %arg1, i32 %arg2, i32 %arg3, i32 %arg4, i64 %arg5) #0 {
; GCN-LABEL: long_branch_hang:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
+; GCN-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s0, 0
-; GCN-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GCN-NEXT: s_cmp_lg_u32 s0, 0
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
; GCN-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GCN-NEXT: s_cmp_lt_i32 s3, 6
+; GCN-NEXT: s_cmp_lg_u32 s2, 0
+; GCN-NEXT: s_cselect_b64 s[10:11], -1, 0
+; GCN-NEXT: s_cmp_lt_i32 s5, 6
; GCN-NEXT: s_cbranch_scc1 .LBB10_1
; GCN-NEXT: ; %bb.8: ; %bb
-; GCN-NEXT: s_getpc_b64 s[8:9]
+; GCN-NEXT: s_getpc_b64 s[10:11]
; GCN-NEXT: .Lpost_getpc12:
-; GCN-NEXT: s_add_u32 s8, s8, (.LBB10_2-.Lpost_getpc12)&4294967295
-; GCN-NEXT: s_addc_u32 s9, s9, (.LBB10_2-.Lpost_getpc12)>>32
-; GCN-NEXT: s_setpc_b64 s[8:9]
+; GCN-NEXT: s_add_u32 s10, s10, (.LBB10_2-.Lpost_getpc12)&4294967295
+; GCN-NEXT: s_addc_u32 s11, s11, (.LBB10_2-.Lpost_getpc12)>>32
+; GCN-NEXT: s_setpc_b64 s[10:11]
; GCN-NEXT: .LBB10_1: ; %bb13
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: v_nop_e64
@@ -1288,63 +1263,60 @@ define amdgpu_kernel void @long_branch_hang(ptr addrspace(1) nocapture %arg, i32
; GCN-NEXT: s_cbranch_execz .LBB10_3
; GCN-NEXT: s_branch .LBB10_4
; GCN-NEXT: .LBB10_2:
-; GCN-NEXT: s_mov_b64 s[8:9], 0
+; GCN-NEXT: s_mov_b64 s[10:11], 0
; GCN-NEXT: .LBB10_3: ; %bb9
-; GCN-NEXT: s_cmp_lt_i32 s3, 11
-; GCN-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GCN-NEXT: s_cmp_ge_i32 s2, s3
+; GCN-NEXT: s_cmp_lt_i32 s5, 11
; GCN-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GCN-NEXT: s_and_b64 s[8:9], s[10:11], s[8:9]
-; GCN-NEXT: .LBB10_4: ; %Flow5
-; GCN-NEXT: s_andn2_b64 vcc, exec, s[8:9]
+; GCN-NEXT: s_cmp_ge_i32 s4, s5
+; GCN-NEXT: s_cselect_b64 s[12:13], -1, 0
+; GCN-NEXT: s_and_b64 s[10:11], s[12:13], s[10:11]
+; GCN-NEXT: .LBB10_4: ; %Flow7
+; GCN-NEXT: s_andn2_b64 vcc, exec, s[10:11]
; GCN-NEXT: s_cbranch_vccz .LBB10_5
-; GCN-NEXT: ; %bb.10: ; %Flow5
-; GCN-NEXT: s_getpc_b64 s[0:1]
+; GCN-NEXT: ; %bb.10: ; %Flow7
+; GCN-NEXT: s_getpc_b64 s[2:3]
; GCN-NEXT: .Lpost_getpc13:
-; GCN-NEXT: s_add_u32 s0, s0, (.LBB10_6-.Lpost_getpc13)&4294967295
-; GCN-NEXT: s_addc_u32 s1, s1, (.LBB10_6-.Lpost_getpc13)>>32
-; GCN-NEXT: s_setpc_b64 s[0:1]
+; GCN-NEXT: s_add_u32 s2, s2, (.LBB10_6-.Lpost_getpc13)&4294967295
+; GCN-NEXT: s_addc_u32 s3, s3, (.LBB10_6-.Lpost_getpc13)>>32
+; GCN-NEXT: s_setpc_b64 s[2:3]
; GCN-NEXT: .LBB10_5: ; %bb14
-; GCN-NEXT: s_cmp_lt_i32 s1, 9
-; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GCN-NEXT: s_cmp_lt_i32 s2, s3
+; GCN-NEXT: s_cmp_lt_i32 s3, 9
; GCN-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
-; GCN-NEXT: s_and_b64 s[0:1], s[6:7], s[0:1]
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GCN-NEXT: s_cmp_lt_i32 s4, s5
+; GCN-NEXT: s_cselect_b64 s[4:5], -1, 0
+; GCN-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
+; GCN-NEXT: s_and_b64 s[2:3], s[8:9], s[2:3]
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
; GCN-NEXT: s_branch .LBB10_7
; GCN-NEXT: .LBB10_6:
; GCN-NEXT: ; implicit-def: $vgpr0
; GCN-NEXT: .LBB10_7: ; %bb19
-; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
; GCN-NEXT: v_mov_b32_e32 v1, s4
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, 0
; GCN-NEXT: v_mov_b32_e32 v2, s5
; GCN-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX11-LABEL: long_branch_hang:
; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
+; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_cmp_eq_u32 s0, 0
-; GFX11-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GFX11-NEXT: s_cmp_lg_u32 s0, 0
+; GFX11-NEXT: s_cmp_eq_u32 s2, 0
; GFX11-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX11-NEXT: s_cmp_lt_i32 s3, 6
+; GFX11-NEXT: s_cmp_lg_u32 s2, 0
+; GFX11-NEXT: s_cselect_b64 s[10:11], -1, 0
+; GFX11-NEXT: s_cmp_lt_i32 s5, 6
; GFX11-NEXT: s_cbranch_scc1 .LBB10_1
; GFX11-NEXT: ; %bb.8: ; %bb
-; GFX11-NEXT: s_getpc_b64 s[8:9]
+; GFX11-NEXT: s_getpc_b64 s[10:11]
; GFX11-NEXT: .Lpost_getpc11:
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_add_u32 s8, s8, (.LBB10_2-.Lpost_getpc11)&4294967295
-; GFX11-NEXT: s_addc_u32 s9, s9, (.LBB10_2-.Lpost_getpc11)>>32
+; GFX11-NEXT: s_add_u32 s10, s10, (.LBB10_2-.Lpost_getpc11)&4294967295
+; GFX11-NEXT: s_addc_u32 s11, s11, (.LBB10_2-.Lpost_getpc11)>>32
; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
-; GFX11-NEXT: s_setpc_b64 s[8:9]
+; GFX11-NEXT: s_setpc_b64 s[10:11]
; GFX11-NEXT: .LBB10_1: ; %bb13
; GFX11-NEXT: ;;#ASMSTART
; GFX11-NEXT: v_nop_e64
@@ -1355,106 +1327,102 @@ define amdgpu_kernel void @long_branch_hang(ptr addrspace(1) nocapture %arg, i32
; GFX11-NEXT: s_cbranch_execz .LBB10_3
; GFX11-NEXT: s_branch .LBB10_4
; GFX11-NEXT: .LBB10_2:
-; GFX11-NEXT: s_mov_b64 s[8:9], 0
+; GFX11-NEXT: s_mov_b64 s[10:11], 0
; GFX11-NEXT: .LBB10_3: ; %bb9
-; GFX11-NEXT: s_cmp_lt_i32 s3, 11
-; GFX11-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GFX11-NEXT: s_cmp_ge_i32 s2, s3
+; GFX11-NEXT: s_cmp_lt_i32 s5, 11
; GFX11-NEXT: s_cselect_b64 s[10:11], -1, 0
+; GFX11-NEXT: s_cmp_ge_i32 s4, s5
+; GFX11-NEXT: s_cselect_b64 s[12:13], -1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_b64 s[8:9], s[10:11], s[8:9]
-; GFX11-NEXT: .LBB10_4: ; %Flow5
+; GFX11-NEXT: s_and_b64 s[10:11], s[12:13], s[10:11]
+; GFX11-NEXT: .LBB10_4: ; %Flow7
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_and_not1_b64 vcc, exec, s[8:9]
+; GFX11-NEXT: s_and_not1_b64 vcc, exec, s[10:11]
; GFX11-NEXT: s_cbranch_vccnz .LBB10_6
; GFX11-NEXT: ; %bb.5: ; %bb14
-; GFX11-NEXT: s_cmp_lt_i32 s1, 9
-; GFX11-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX11-NEXT: s_cmp_lt_i32 s2, s3
+; GFX11-NEXT: s_cmp_lt_i32 s3, 9
; GFX11-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX11-NEXT: s_cmp_lt_i32 s4, s5
+; GFX11-NEXT: s_cselect_b64 s[4:5], -1, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
-; GFX11-NEXT: s_and_b64 s[0:1], s[6:7], s[0:1]
+; GFX11-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
+; GFX11-NEXT: s_and_b64 s[2:3], s[8:9], s[2:3]
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; GFX11-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
; GFX11-NEXT: s_branch .LBB10_7
; GFX11-NEXT: .LBB10_6:
; GFX11-NEXT: ; implicit-def: $vgpr0
; GFX11-NEXT: .LBB10_7: ; %bb19
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x3c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_lshl_b64 s[2:3], s[6:7], 2
+; GFX11-NEXT: s_waitcnt_depctr depctr_sa_sdst(0)
; GFX11-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_add_u32 s0, s0, s2
+; GFX11-NEXT: s_addc_u32 s1, s1, s3
; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: long_branch_hang:
; GFX12: ; %bb.0: ; %bb
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-NEXT: s_mov_b32 s7, -1
+; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
+; GFX12-NEXT: s_mov_b32 s9, -1
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_cmp_eq_u32 s0, 0
-; GFX12-NEXT: s_cselect_b32 s6, -1, 0
-; GFX12-NEXT: s_cmp_lg_u32 s0, 0
-; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_cmp_eq_u32 s2, 0
; GFX12-NEXT: s_cselect_b32 s8, -1, 0
-; GFX12-NEXT: s_cmp_lt_i32 s3, 6
+; GFX12-NEXT: s_cmp_lg_u32 s2, 0
+; GFX12-NEXT: s_mov_b32 s2, 0
+; GFX12-NEXT: s_cselect_b32 s10, -1, 0
+; GFX12-NEXT: s_cmp_lt_i32 s5, 6
; GFX12-NEXT: s_cbranch_scc0 .LBB10_1
; GFX12-NEXT: ; %bb.18: ; %bb
-; GFX12-NEXT: s_getpc_b64 s[10:11]
-; GFX12-NEXT: .Lpost_getpc17:
+; GFX12-NEXT: s_getpc_b64 s[12:13]
+; GFX12-NEXT: .Lpost_getpc16:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s10, s10, (.LBB10_4-.Lpost_getpc17)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s11, s11, (.LBB10_4-.Lpost_getpc17)>>32
+; GFX12-NEXT: s_add_co_u32 s12, s12, (.LBB10_4-.Lpost_getpc16)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s13, s13, (.LBB10_4-.Lpost_getpc16)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[10:11]
+; GFX12-NEXT: s_setpc_b64 s[12:13]
; GFX12-NEXT: .LBB10_1: ; %Flow
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s7
+; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s9
; GFX12-NEXT: s_cbranch_vccnz .LBB10_2
; GFX12-NEXT: ; %bb.10: ; %Flow
-; GFX12-NEXT: s_getpc_b64 s[8:9]
-; GFX12-NEXT: .Lpost_getpc13:
+; GFX12-NEXT: s_getpc_b64 s[10:11]
+; GFX12-NEXT: .Lpost_getpc12:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s8, s8, (.LBB10_5-.Lpost_getpc13)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s9, s9, (.LBB10_5-.Lpost_getpc13)>>32
+; GFX12-NEXT: s_add_co_u32 s10, s10, (.LBB10_5-.Lpost_getpc12)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s11, s11, (.LBB10_5-.Lpost_getpc12)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[8:9]
-; GFX12-NEXT: .LBB10_2: ; %Flow5
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
+; GFX12-NEXT: s_setpc_b64 s[10:11]
+; GFX12-NEXT: .LBB10_2: ; %Flow7
+; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
; GFX12-NEXT: s_cbranch_vccz .LBB10_3
-; GFX12-NEXT: ; %bb.12: ; %Flow5
-; GFX12-NEXT: s_getpc_b64 s[0:1]
-; GFX12-NEXT: .Lpost_getpc14:
+; GFX12-NEXT: ; %bb.12: ; %Flow7
+; GFX12-NEXT: s_getpc_b64 s[2:3]
+; GFX12-NEXT: .Lpost_getpc13:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB10_6-.Lpost_getpc14)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB10_6-.Lpost_getpc14)>>32
+; GFX12-NEXT: s_add_co_u32 s2, s2, (.LBB10_6-.Lpost_getpc13)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s3, s3, (.LBB10_6-.Lpost_getpc13)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[0:1]
+; GFX12-NEXT: s_setpc_b64 s[2:3]
; GFX12-NEXT: .LBB10_3: ; %bb14
-; GFX12-NEXT: s_cmp_lt_i32 s1, 9
-; GFX12-NEXT: s_cselect_b32 s0, -1, 0
-; GFX12-NEXT: s_cmp_lt_i32 s2, s3
-; GFX12-NEXT: s_cselect_b32 s1, -1, 0
+; GFX12-NEXT: s_cmp_lt_i32 s3, 9
+; GFX12-NEXT: s_cselect_b32 s2, -1, 0
+; GFX12-NEXT: s_cmp_lt_i32 s4, s5
+; GFX12-NEXT: s_cselect_b32 s3, -1, 0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_or_b32 s0, s1, s0
-; GFX12-NEXT: s_and_b32 s0, s6, s0
+; GFX12-NEXT: s_or_b32 s2, s3, s2
+; GFX12-NEXT: s_and_b32 s2, s8, s2
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_cndmask_b32_e64 v0, 0, 1, s0
+; GFX12-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2
; GFX12-NEXT: ; %bb.8: ; %bb14
-; GFX12-NEXT: s_getpc_b64 s[0:1]
-; GFX12-NEXT: .Lpost_getpc12:
+; GFX12-NEXT: s_getpc_b64 s[2:3]
+; GFX12-NEXT: .Lpost_getpc11:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s0, s0, (.LBB10_7-.Lpost_getpc12)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s1, s1, (.LBB10_7-.Lpost_getpc12)>>32
+; GFX12-NEXT: s_add_co_u32 s2, s2, (.LBB10_7-.Lpost_getpc11)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s3, s3, (.LBB10_7-.Lpost_getpc11)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[0:1]
+; GFX12-NEXT: s_setpc_b64 s[2:3]
; GFX12-NEXT: .LBB10_4: ; %bb13
-; GFX12-NEXT: s_mov_b32 s0, s8
+; GFX12-NEXT: s_mov_b32 s2, s10
; GFX12-NEXT: ;;#ASMSTART
; GFX12-NEXT: v_nop_e64
; GFX12-NEXT: v_nop_e64
@@ -1463,41 +1431,37 @@ define amdgpu_kernel void @long_branch_hang(ptr addrspace(1) nocapture %arg, i32
; GFX12-NEXT: ;;#ASMEND
; GFX12-NEXT: s_cbranch_execz .LBB10_5
; GFX12-NEXT: ; %bb.14: ; %bb13
-; GFX12-NEXT: s_getpc_b64 s[8:9]
-; GFX12-NEXT: .Lpost_getpc15:
+; GFX12-NEXT: s_getpc_b64 s[10:11]
+; GFX12-NEXT: .Lpost_getpc14:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s8, s8, (.LBB10_2-.Lpost_getpc15)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s9, s9, (.LBB10_2-.Lpost_getpc15)>>32
+; GFX12-NEXT: s_add_co_u32 s10, s10, (.LBB10_2-.Lpost_getpc14)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s11, s11, (.LBB10_2-.Lpost_getpc14)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[8:9]
+; GFX12-NEXT: s_setpc_b64 s[10:11]
; GFX12-NEXT: .LBB10_5: ; %bb9
-; GFX12-NEXT: s_cmp_lt_i32 s3, 11
-; GFX12-NEXT: s_cselect_b32 s0, -1, 0
-; GFX12-NEXT: s_cmp_ge_i32 s2, s3
-; GFX12-NEXT: s_cselect_b32 s7, -1, 0
+; GFX12-NEXT: s_cmp_lt_i32 s5, 11
+; GFX12-NEXT: s_cselect_b32 s2, -1, 0
+; GFX12-NEXT: s_cmp_ge_i32 s4, s5
+; GFX12-NEXT: s_cselect_b32 s9, -1, 0
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_and_b32 s0, s7, s0
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s0
+; GFX12-NEXT: s_and_b32 s2, s9, s2
+; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s2
; GFX12-NEXT: s_cbranch_vccnz .LBB10_6
; GFX12-NEXT: ; %bb.16: ; %bb9
-; GFX12-NEXT: s_getpc_b64 s[8:9]
-; GFX12-NEXT: .Lpost_getpc16:
+; GFX12-NEXT: s_getpc_b64 s[10:11]
+; GFX12-NEXT: .Lpost_getpc15:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_co_u32 s8, s8, (.LBB10_3-.Lpost_getpc16)&4294967295
-; GFX12-NEXT: s_add_co_ci_u32 s9, s9, (.LBB10_3-.Lpost_getpc16)>>32
+; GFX12-NEXT: s_add_co_u32 s10, s10, (.LBB10_3-.Lpost_getpc15)&4294967295
+; GFX12-NEXT: s_add_co_ci_u32 s11, s11, (.LBB10_3-.Lpost_getpc15)>>32
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_setpc_b64 s[8:9]
+; GFX12-NEXT: s_setpc_b64 s[10:11]
; GFX12-NEXT: .LBB10_6:
; GFX12-NEXT: ; implicit-def: $vgpr0
; GFX12-NEXT: .LBB10_7: ; %bb19
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x3c
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GFX12-NEXT: s_lshl_b64 s[2:3], s[6:7], 2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_add_nc_u64 s[0:1], s[2:3], s[0:1]
+; GFX12-NEXT: s_add_nc_u64 s[0:1], s[0:1], s[2:3]
; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-NEXT: s_endpgm
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll b/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
index 20476efbaa6fa..85270ce6d3293 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-intrinsics-mmo-offsets.ll
@@ -11,29 +11,22 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_32 = COPY $sgpr0
; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_64 = REG_SEQUENCE [[COPY1]], %subreg.sub0, [[S_MOV_B32_]], %subreg.sub1
- ; GCN-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[REG_SEQUENCE]], 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg0, addrspace 6)
- ; GCN-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[REG_SEQUENCE]], 0, 0 :: (dereferenceable invariant load (s64) from %ir.arg0, align 16, addrspace 6)
- ; GCN-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[REG_SEQUENCE]], 8, 0 :: (dereferenceable invariant load (s64) from %ir.arg0 + 8, basealign 16, addrspace 6)
- ; GCN-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub1
- ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM1]].sub0
- ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
- ; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub0
- ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1, killed [[COPY3]], %subreg.sub2, killed [[COPY2]], %subreg.sub3
+ ; GCN-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM killed [[REG_SEQUENCE]], 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg0, addrspace 6)
; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 64
; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_1]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 128
; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY6]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY2]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_1]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[REG_SEQUENCE1]], [[COPY7]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_1]], 64, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY3]], 128, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 72
@@ -41,15 +34,15 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
; GCN-NEXT: [[S_MOV_B32_4:%[0-9]+]]:sreg_32 = S_MOV_B32 144
; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY8]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY4]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_3]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE1]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[REG_SEQUENCE1]], [[COPY9]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_3]], 72, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_OFFSET [[S_LOAD_DWORDX4_IMM]], [[COPY5]], 144, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
; GCN-NEXT: [[S_MOV_B32_5:%[0-9]+]]:sreg_32 = S_MOV_B32 80
@@ -57,32 +50,32 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
; GCN-NEXT: [[S_MOV_B32_6:%[0-9]+]]:sreg_32 = S_MOV_B32 160
; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: [[COPY10:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[COPY10]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[COPY6]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_5]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[COPY11:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[REG_SEQUENCE1]], [[COPY11]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_5]], 80, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_6]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_OFFSET [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[COPY7]], 160, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY]], %subreg.sub1
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[COPY]], %subreg.sub0, [[COPY]], %subreg.sub1
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
; GCN-NEXT: [[S_MOV_B32_7:%[0-9]+]]:sreg_32 = S_MOV_B32 88
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_7]], 88, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_7]], 88, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
; GCN-NEXT: [[S_MOV_B32_8:%[0-9]+]]:sreg_32 = S_MOV_B32 176
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE2]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: [[COPY12:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[COPY12]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[COPY8]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[S_MOV_B32_7]], 88, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE2]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[COPY13:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE2]], [[REG_SEQUENCE1]], [[COPY13]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_7]], 88, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_8]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_OFFSET [[REG_SEQUENCE1]], [[S_LOAD_DWORDX4_IMM]], [[COPY9]], 176, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
; GCN-NEXT: [[S_MOV_B32_9:%[0-9]+]]:sreg_32 = S_MOV_B32 96
@@ -90,15 +83,15 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
; GCN-NEXT: [[S_MOV_B32_10:%[0-9]+]]:sreg_32 = S_MOV_B32 192
; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[COPY14:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[COPY14]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY10:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[COPY10]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET4]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET5]], [[REG_SEQUENCE1]], [[S_MOV_B32_9]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET6]], [[REG_SEQUENCE1]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN1]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[COPY15:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET7]], [[REG_SEQUENCE1]], [[COPY15]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET5]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_9]], 96, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET6]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFEN_exact killed [[BUFFER_LOAD_DWORDX4_OFFEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_10]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY11:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_OFFSET_exact killed [[BUFFER_LOAD_DWORDX4_OFFSET7]], [[S_LOAD_DWORDX4_IMM]], [[COPY11]], 192, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
; GCN-NEXT: [[S_MOV_B32_11:%[0-9]+]]:sreg_32 = S_MOV_B32 104
@@ -106,182 +99,182 @@ define amdgpu_cs void @mmo_offsets0(ptr addrspace(6) inreg noalias align(16) der
; GCN-NEXT: [[S_MOV_B32_12:%[0-9]+]]:sreg_32 = S_MOV_B32 208
; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[COPY16:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[COPY16]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY12:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET3]], [[S_LOAD_DWORDX4_IMM]], [[COPY12]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5]], [[REG_SEQUENCE1]], [[S_MOV_B32_11]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6]], [[REG_SEQUENCE1]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[COPY17:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7]], [[REG_SEQUENCE1]], [[COPY17]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET4]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET5]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_11]], 104, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET6]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFEN1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_12]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY13:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_OFFSET_exact killed [[BUFFER_LOAD_FORMAT_XYZW_OFFSET7]], [[S_LOAD_DWORDX4_IMM]], [[COPY13]], 208, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[COPY18:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY18]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY14]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: [[S_MOV_B32_13:%[0-9]+]]:sreg_32 = S_MOV_B32 112
- ; GCN-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY19]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_13]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY15:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY15]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_13]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: [[S_MOV_B32_14:%[0-9]+]]:sreg_32 = S_MOV_B32 224
- ; GCN-NEXT: [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY20]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[COPY]], %subreg.sub1
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY22:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY21]], [[S_LOAD_DWORDX4_IMM]], [[COPY22]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY16:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY16]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:vreg_64 = REG_SEQUENCE [[S_MOV_B32_]], %subreg.sub0, [[COPY]], %subreg.sub1
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY17:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY18:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY17]], [[S_LOAD_DWORDX4_IMM]], [[COPY18]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec
; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[COPY23:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY23]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY19:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY19]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY20:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY20]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_13]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY21:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY21]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_BOTHEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY22:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY23:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY22]], [[S_LOAD_DWORDX4_IMM]], [[COPY23]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: INLINEASM &"", sideeffect attdialect
; GCN-NEXT: [[COPY24:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY24]], [[REG_SEQUENCE1]], [[S_MOV_B32_13]], 112, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY24]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[S_MOV_B32_15:%[0-9]+]]:sreg_32 = S_MOV_B32 120
; GCN-NEXT: [[COPY25:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY25]], [[REG_SEQUENCE1]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_BOTHEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_BOTHEN [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_14]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY25]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_15]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[S_MOV_B32_16:%[0-9]+]]:sreg_32 = S_MOV_B32 240
; GCN-NEXT: [[COPY26:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY27:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY26]], [[REG_SEQUENCE1]], [[COPY27]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_DWORDX4_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_DWORDX4_IDXEN [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 224, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY26]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY27:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY28:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY27]], [[S_LOAD_DWORDX4_IMM]], [[COPY28]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[COPY28:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY28]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[S_MOV_B32_15:%[0-9]+]]:sreg_32 = S_MOV_B32 120
; GCN-NEXT: [[COPY29:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY29]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_15]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[S_MOV_B32_16:%[0-9]+]]:sreg_32 = S_MOV_B32 240
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY29]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
; GCN-NEXT: [[COPY30:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY30]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY30]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_15]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
; GCN-NEXT: [[COPY31:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY32:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY31]], [[S_LOAD_DWORDX4_IMM]], [[COPY32]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY31]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY32:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY33:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY32]], [[S_LOAD_DWORDX4_IMM]], [[COPY33]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.2, align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[COPY33:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY33]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
; GCN-NEXT: [[COPY34:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY34]], [[REG_SEQUENCE1]], [[S_MOV_B32_15]], 120, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY34]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
; GCN-NEXT: [[COPY35:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY35]], [[REG_SEQUENCE1]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN1:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_BOTHEN [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_16]], 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY35]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 128, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: [[S_MOV_B32_17:%[0-9]+]]:sreg_32 = S_MOV_B32 256
; GCN-NEXT: [[COPY36:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY37:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY36]], [[REG_SEQUENCE1]], [[COPY37]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11:%[0-9]+]]:vreg_128 = BUFFER_LOAD_FORMAT_XYZW_IDXEN [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 240, 0, 0, implicit $exec :: (dereferenceable load (s128) from %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY36]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_BOTHEN [[COPY]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY37:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY38:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY37]], [[S_LOAD_DWORDX4_IMM]], [[COPY38]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[COPY38:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY38]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
; GCN-NEXT: [[COPY39:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY39]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 128, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: [[S_MOV_B32_17:%[0-9]+]]:sreg_32 = S_MOV_B32 256
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY39]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
; GCN-NEXT: [[COPY40:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY40]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_BOTHEN [[COPY]], [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY40]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_2]], 128, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
; GCN-NEXT: [[COPY41:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY42:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY41]], [[S_LOAD_DWORDX4_IMM]], [[COPY42]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY41]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_BOTHEN [[COPY]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY42:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY43:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY42]], [[S_LOAD_DWORDX4_IMM]], [[COPY43]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[COPY43:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY43]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
; GCN-NEXT: [[COPY44:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY44]], [[REG_SEQUENCE1]], [[S_MOV_B32_2]], 128, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY44]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: [[S_MOV_B32_18:%[0-9]+]]:sreg_32 = S_MOV_B32 136
; GCN-NEXT: [[COPY45:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY45]], [[REG_SEQUENCE1]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_BOTHEN [[COPY]], [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_17]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY45]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_18]], 136, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: [[S_MOV_B32_19:%[0-9]+]]:sreg_32 = S_MOV_B32 272
; GCN-NEXT: [[COPY46:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY47:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY46]], [[REG_SEQUENCE1]], [[COPY47]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_ADD_IDXEN [[COPY]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 256, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY46]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY47:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY48:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY47]], [[S_LOAD_DWORDX4_IMM]], [[COPY48]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[COPY48:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY48]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: [[S_MOV_B32_18:%[0-9]+]]:sreg_32 = S_MOV_B32 136
; GCN-NEXT: [[COPY49:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY49]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_18]], 136, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: [[S_MOV_B32_19:%[0-9]+]]:sreg_32 = S_MOV_B32 272
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY49]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
; GCN-NEXT: [[COPY50:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY50]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY50]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_18]], 136, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
; GCN-NEXT: [[COPY51:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY52:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY51]], [[S_LOAD_DWORDX4_IMM]], [[COPY52]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY51]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE1]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY52:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY53:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY52]], [[S_LOAD_DWORDX4_IMM]], [[COPY53]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE1]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.2, align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[COPY53:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY53]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
; GCN-NEXT: [[COPY54:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY54]], [[REG_SEQUENCE1]], [[S_MOV_B32_18]], 136, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN]], [[COPY54]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
; GCN-NEXT: [[COPY55:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY55]], [[REG_SEQUENCE1]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_BOTHEN [[REG_SEQUENCE2]], [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_19]], 0, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN1]], [[COPY55]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[S_MOV_B32_20:%[0-9]+]]:sreg_32 = S_MOV_B32 288
; GCN-NEXT: [[COPY56:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY57:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY56]], [[REG_SEQUENCE1]], [[COPY57]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_ATOMIC_CMPSWAP_IDXEN [[REG_SEQUENCE2]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 272, 0, implicit $exec :: (volatile dereferenceable load store (s32) on %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN2]], [[COPY56]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY57:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY58:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN3]], [[COPY57]], [[S_LOAD_DWORDX4_IMM]], [[COPY58]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN4]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN5]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[COPY58:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN]], [[COPY58]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
; GCN-NEXT: [[COPY59:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN1]], [[COPY59]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[S_MOV_B32_20:%[0-9]+]]:sreg_32 = S_MOV_B32 288
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN6]], [[COPY59]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
; GCN-NEXT: [[COPY60:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN2]], [[COPY60]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN]], [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN7]], [[COPY60]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_4]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
; GCN-NEXT: [[COPY61:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY62:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN3]], [[COPY61]], [[S_LOAD_DWORDX4_IMM]], [[COPY62]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN4]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN5]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN8]], [[COPY61]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN1]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY62:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY63:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN9]], [[COPY62]], [[S_LOAD_DWORDX4_IMM]], [[COPY63]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN10]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN11]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[COPY63:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN6]], [[COPY63]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
; GCN-NEXT: [[COPY64:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN7]], [[COPY64]], [[REG_SEQUENCE1]], [[S_MOV_B32_4]], 144, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN]], [[COPY64]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[S_MOV_B32_21:%[0-9]+]]:sreg_32 = S_MOV_B32 152
; GCN-NEXT: [[COPY65:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN8]], [[COPY65]], [[REG_SEQUENCE1]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_BOTHEN_exact killed [[BUFFER_LOAD_DWORDX4_BOTHEN1]], [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_20]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1]], [[COPY65]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_21]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[S_MOV_B32_22:%[0-9]+]]:sreg_32 = S_MOV_B32 304
; GCN-NEXT: [[COPY66:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY67:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN9]], [[COPY66]], [[REG_SEQUENCE1]], [[COPY67]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN10]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_DWORDX4_IDXEN_exact killed [[BUFFER_LOAD_DWORDX4_IDXEN11]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 288, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: INLINEASM &"", sideeffect attdialect
- ; GCN-NEXT: [[COPY68:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN]], [[COPY68]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[S_MOV_B32_21:%[0-9]+]]:sreg_32 = S_MOV_B32 152
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2]], [[COPY66]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY67:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY68:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3]], [[COPY67]], [[S_LOAD_DWORDX4_IMM]], [[COPY68]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
; GCN-NEXT: [[COPY69:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN1]], [[COPY69]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_21]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[S_MOV_B32_22:%[0-9]+]]:sreg_32 = S_MOV_B32 304
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6]], [[COPY69]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
; GCN-NEXT: [[COPY70:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN2]], [[COPY70]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN]], [[REG_SEQUENCE3]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7]], [[COPY70]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_21]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
; GCN-NEXT: [[COPY71:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY72:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN3]], [[COPY71]], [[S_LOAD_DWORDX4_IMM]], [[COPY72]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN4]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN5]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128), align 1, addrspace 8)
- ; GCN-NEXT: [[COPY73:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN6]], [[COPY73]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[COPY74:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN7]], [[COPY74]], [[REG_SEQUENCE1]], [[S_MOV_B32_21]], 152, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[COPY75:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8]], [[COPY75]], [[REG_SEQUENCE1]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN1]], [[REG_SEQUENCE3]], [[REG_SEQUENCE1]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: [[COPY76:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
- ; GCN-NEXT: [[COPY77:%[0-9]+]]:sreg_32 = COPY [[COPY]]
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9]], [[COPY76]], [[REG_SEQUENCE1]], [[COPY77]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10]], [[V_MOV_B32_e32_]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
- ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11]], [[COPY]], [[REG_SEQUENCE1]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.tmp1, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN8]], [[COPY71]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_BOTHEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_BOTHEN1]], [[REG_SEQUENCE2]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_22]], 0, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: [[COPY72:%[0-9]+]]:vgpr_32 = COPY [[S_MOV_B32_]]
+ ; GCN-NEXT: [[COPY73:%[0-9]+]]:sreg_32 = COPY [[COPY]]
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN9]], [[COPY72]], [[S_LOAD_DWORDX4_IMM]], [[COPY73]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN10]], [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
+ ; GCN-NEXT: BUFFER_STORE_FORMAT_XYZW_IDXEN_exact killed [[BUFFER_LOAD_FORMAT_XYZW_IDXEN11]], [[COPY]], [[S_LOAD_DWORDX4_IMM]], [[S_MOV_B32_]], 304, 0, 0, implicit $exec :: (dereferenceable store (s128) into %ir.2, align 1, addrspace 8)
; GCN-NEXT: S_ENDPGM 0
bb.0:
%tmp0 = load <4 x i32>, ptr addrspace(6) %arg0, align 16, !invariant.load !0
diff --git a/llvm/test/CodeGen/AMDGPU/build_vector.ll b/llvm/test/CodeGen/AMDGPU/build_vector.ll
index 37f4094806637..dacc34ca71917 100644
--- a/llvm/test/CodeGen/AMDGPU/build_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/build_vector.ll
@@ -184,35 +184,33 @@ entry:
define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32 %a) {
; GFX6-LABEL: build_vector_v2i16_trunc:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s7, 5
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_lshr_b64 s[4:5], s[6:7], 16
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s3, 5
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_lshr_b64 s[0:1], s[2:3], 16
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: build_vector_v2i16_trunc:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s2, s2, 16
-; GFX8-NEXT: s_or_b32 s2, s2, 0x50000
; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_lshr_b32 s0, s2, 16
+; GFX8-NEXT: s_or_b32 s0, s0, 0x50000
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX10-LABEL: build_vector_v2i16_trunc:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_lshr_b32 s2, s2, 16
@@ -223,9 +221,7 @@ define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32
;
; GFX11-LABEL: build_vector_v2i16_trunc:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_pack_hl_b32_b16 s2, s2, 5
@@ -236,8 +232,7 @@ define amdgpu_kernel void @build_vector_v2i16_trunc (ptr addrspace(1) %out, i32
;
; GFX942-LABEL: build_vector_v2i16_trunc:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_lshr_b32 s2, s2, 16
@@ -257,28 +252,29 @@ define amdgpu_kernel void @build_v2i32_from_v4i16_shuffle(ptr addrspace(1) %out,
; GFX6-LABEL: build_v2i32_from_v4i16_shuffle:
; GFX6: ; %bb.0: ; %entry
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_lshl_b32 s5, s5, 16
-; GFX6-NEXT: s_lshl_b32 s4, s4, 16
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: v_mov_b32_e32 v1, s5
-; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_lshl_b32 s0, s3, 16
+; GFX6-NEXT: s_lshl_b32 s1, s2, 16
+; GFX6-NEXT: v_mov_b32_e32 v0, s1
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: build_v2i32_from_v4i16_shuffle:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshl_b32 s3, s3, 16
-; GFX8-NEXT: s_lshl_b32 s2, s2, 16
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: s_lshl_b32 s0, s3, 16
+; GFX8-NEXT: s_lshl_b32 s1, s2, 16
+; GFX8-NEXT: v_mov_b32_e32 v2, s1
+; GFX8-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX10-LABEL: build_v2i32_from_v4i16_shuffle:
diff --git a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
index 0b2452cf11798..bae6bcf32ae27 100644
--- a/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
+++ b/llvm/test/CodeGen/AMDGPU/carryout-selection.ll
@@ -849,25 +849,25 @@ define amdgpu_kernel void @uaddo32_vcc_user(ptr addrspace(1) %out, ptr addrspace
; GCN-ISEL-NEXT: {{ $}}
; GCN-ISEL-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; GCN-ISEL-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 9, 0 :: (dereferenceable invariant load (s128) from %ir.out.kernarg.offset, align 4, addrspace 4)
-; GCN-ISEL-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 13, 0 :: (dereferenceable invariant load (s64) from %ir.a.kernarg.offset, align 4, addrspace 4)
-; GCN-ISEL-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub1
-; GCN-ISEL-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
-; GCN-ISEL-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, killed [[COPY1]], %subreg.sub1
-; GCN-ISEL-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
-; GCN-ISEL-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+; GCN-ISEL-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 13, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset + 16, align 4, addrspace 4)
+; GCN-ISEL-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+; GCN-ISEL-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub0
+; GCN-ISEL-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
+; GCN-ISEL-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
+; GCN-ISEL-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub1
+; GCN-ISEL-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
+; GCN-ISEL-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY6]], %subreg.sub0, killed [[COPY5]], %subreg.sub1
+; GCN-ISEL-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+; GCN-ISEL-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
; GCN-ISEL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GCN-ISEL-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-; GCN-ISEL-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY4]], %subreg.sub0, killed [[COPY3]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
-; GCN-ISEL-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
-; GCN-ISEL-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
-; GCN-ISEL-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY6]], %subreg.sub0, killed [[COPY5]], %subreg.sub1
-; GCN-ISEL-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE2]].sub1
-; GCN-ISEL-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE2]].sub0
-; GCN-ISEL-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
-; GCN-ISEL-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub0
-; GCN-ISEL-NEXT: [[COPY10:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
-; GCN-ISEL-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY killed [[COPY10]]
-; GCN-ISEL-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 killed [[COPY9]], [[COPY11]], 0, implicit $exec
+; GCN-ISEL-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
+; GCN-ISEL-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY4]], %subreg.sub0, killed [[COPY3]], %subreg.sub1
+; GCN-ISEL-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE2]].sub1
+; GCN-ISEL-NEXT: [[COPY10:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE2]].sub0
+; GCN-ISEL-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY10]], %subreg.sub0, killed [[COPY9]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
+; GCN-ISEL-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY killed [[COPY1]]
+; GCN-ISEL-NEXT: [[V_ADD_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_ADD_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_ADD_CO_U32_e64 killed [[COPY2]], [[COPY11]], 0, implicit $exec
; GCN-ISEL-NEXT: BUFFER_STORE_DWORD_OFFSET killed [[V_ADD_CO_U32_e64_]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.2, addrspace 1)
; GCN-ISEL-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[V_ADD_CO_U32_e64_1]], implicit $exec
; GCN-ISEL-NEXT: BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.3, addrspace 1)
@@ -2098,25 +2098,25 @@ define amdgpu_kernel void @usubo32_vcc_user(ptr addrspace(1) %out, ptr addrspace
; GCN-ISEL-NEXT: {{ $}}
; GCN-ISEL-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; GCN-ISEL-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 9, 0 :: (dereferenceable invariant load (s128) from %ir.out.kernarg.offset, align 4, addrspace 4)
-; GCN-ISEL-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 13, 0 :: (dereferenceable invariant load (s64) from %ir.a.kernarg.offset, align 4, addrspace 4)
-; GCN-ISEL-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub1
-; GCN-ISEL-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
-; GCN-ISEL-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, killed [[COPY1]], %subreg.sub1
-; GCN-ISEL-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
-; GCN-ISEL-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+; GCN-ISEL-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 13, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset + 16, align 4, addrspace 4)
+; GCN-ISEL-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
+; GCN-ISEL-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub0
+; GCN-ISEL-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
+; GCN-ISEL-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
+; GCN-ISEL-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub1
+; GCN-ISEL-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
+; GCN-ISEL-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY6]], %subreg.sub0, killed [[COPY5]], %subreg.sub1
+; GCN-ISEL-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+; GCN-ISEL-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
; GCN-ISEL-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GCN-ISEL-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
-; GCN-ISEL-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY4]], %subreg.sub0, killed [[COPY3]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
-; GCN-ISEL-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
-; GCN-ISEL-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
-; GCN-ISEL-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY6]], %subreg.sub0, killed [[COPY5]], %subreg.sub1
-; GCN-ISEL-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE2]].sub1
-; GCN-ISEL-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE2]].sub0
-; GCN-ISEL-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
-; GCN-ISEL-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub0
-; GCN-ISEL-NEXT: [[COPY10:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
-; GCN-ISEL-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY killed [[COPY10]]
-; GCN-ISEL-NEXT: [[V_SUB_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_SUB_CO_U32_e64 killed [[COPY9]], [[COPY11]], 0, implicit $exec
+; GCN-ISEL-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY8]], %subreg.sub0, killed [[COPY7]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
+; GCN-ISEL-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY4]], %subreg.sub0, killed [[COPY3]], %subreg.sub1
+; GCN-ISEL-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE2]].sub1
+; GCN-ISEL-NEXT: [[COPY10:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE2]].sub0
+; GCN-ISEL-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY10]], %subreg.sub0, killed [[COPY9]], %subreg.sub1, [[S_MOV_B32_1]], %subreg.sub2, [[S_MOV_B32_]], %subreg.sub3
+; GCN-ISEL-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY killed [[COPY1]]
+; GCN-ISEL-NEXT: [[V_SUB_CO_U32_e64_:%[0-9]+]]:vgpr_32, [[V_SUB_CO_U32_e64_1:%[0-9]+]]:sreg_64_xexec = V_SUB_CO_U32_e64 killed [[COPY2]], [[COPY11]], 0, implicit $exec
; GCN-ISEL-NEXT: BUFFER_STORE_DWORD_OFFSET killed [[V_SUB_CO_U32_e64_]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (store (s32) into %ir.2, addrspace 1)
; GCN-ISEL-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[V_SUB_CO_U32_e64_1]], implicit $exec
; GCN-ISEL-NEXT: BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE3]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.3, addrspace 1)
@@ -3929,7 +3929,7 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-ISEL-NEXT: S_CBRANCH_SCC1 %bb.4, implicit $scc
; GCN-ISEL-NEXT: S_BRANCH %bb.2
; GCN-ISEL-NEXT: {{ $}}
-; GCN-ISEL-NEXT: bb.2 (%ir-block.7):
+; GCN-ISEL-NEXT: bb.2 (%ir-block.9):
; GCN-ISEL-NEXT: successors: %bb.4(0x80000000)
; GCN-ISEL-NEXT: {{ $}}
; GCN-ISEL-NEXT: [[COPY12:%[0-9]+]]:sreg_32 = COPY [[COPY9]].sub0
@@ -3967,7 +3967,7 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-ISEL-NEXT: [[COPY21:%[0-9]+]]:sreg_64 = COPY [[REG_SEQUENCE3]]
; GCN-ISEL-NEXT: S_BRANCH %bb.4
; GCN-ISEL-NEXT: {{ $}}
-; GCN-ISEL-NEXT: bb.3 (%ir-block.12):
+; GCN-ISEL-NEXT: bb.3 (%ir-block.14):
; GCN-ISEL-NEXT: successors: %bb.1(0x80000000)
; GCN-ISEL-NEXT: {{ $}}
; GCN-ISEL-NEXT: [[COPY22:%[0-9]+]]:sreg_32 = COPY [[COPY9]].sub0
@@ -4167,7 +4167,7 @@ define amdgpu_kernel void @sudiv64(ptr addrspace(1) %out, i64 %x, i64 %y) {
; GCN-ISEL-NEXT: [[S_MOV_B32_13:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
; GCN-ISEL-NEXT: [[REG_SEQUENCE25:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY83]], %subreg.sub0, killed [[COPY82]], %subreg.sub1, killed [[S_MOV_B32_13]], %subreg.sub2, killed [[S_MOV_B32_12]], %subreg.sub3
; GCN-ISEL-NEXT: [[COPY84:%[0-9]+]]:vreg_64 = COPY [[PHI2]]
-; GCN-ISEL-NEXT: BUFFER_STORE_DWORDX2_OFFSET [[COPY84]], killed [[REG_SEQUENCE25]], 0, 0, 0, 0, implicit $exec :: (store (s64) into %ir.15, addrspace 1)
+; GCN-ISEL-NEXT: BUFFER_STORE_DWORDX2_OFFSET [[COPY84]], killed [[REG_SEQUENCE25]], 0, 0, 0, 0, implicit $exec :: (store (s64) into %ir.17, addrspace 1)
; GCN-ISEL-NEXT: S_ENDPGM 0
%result = udiv i64 %x, %y
store i64 %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
index ac121195de432..d5393e558233a 100644
--- a/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
+++ b/llvm/test/CodeGen/AMDGPU/coexec-scheduler.ll
@@ -9,8 +9,8 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; COEXEC-NEXT: v_mov_b32_e32 v0, 0
; COEXEC-NEXT: s_clause 0x1
-; COEXEC-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
-; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
+; COEXEC-NEXT: s_load_b32 s1, s[4:5], 0x0 nv
+; COEXEC-NEXT: s_load_b128 s[8:11], s[4:5], 0x8 nv
; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; COEXEC-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; COEXEC-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
@@ -26,7 +26,7 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: v_dual_mov_b32 v25, v0 :: v_dual_mov_b32 v26, v0
; COEXEC-NEXT: v_dual_mov_b32 v27, v0 :: v_dual_mov_b32 v28, v0
; COEXEC-NEXT: s_wait_kmcnt 0x0
-; COEXEC-NEXT: s_bitcmp1_b32 s0, 0
+; COEXEC-NEXT: s_bitcmp1_b32 s10, 0
; COEXEC-NEXT: v_mov_b32_e32 v29, v0
; COEXEC-NEXT: s_cselect_b32 s0, -1, 0
; COEXEC-NEXT: v_mov_b32_e32 v30, v0
@@ -41,8 +41,8 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_mov_b32_e32 v88, s2
-; COEXEC-NEXT: s_add_co_i32 s2, s2, s1
+; COEXEC-NEXT: v_mov_b32_e32 v88, s1
+; COEXEC-NEXT: s_add_co_i32 s1, s1, s11
; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v88 offset:192
; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v88
; COEXEC-NEXT: ds_load_tr16_b128 v[44:47], v88 offset:64
@@ -75,17 +75,15 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_mov_b32_e32 v32, 0
-; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
-; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
-; COEXEC-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
-; COEXEC-NEXT: global_store_b128 v32, v[24:27], s[0:1]
-; COEXEC-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
-; COEXEC-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
-; COEXEC-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
-; COEXEC-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
-; COEXEC-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
-; COEXEC-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
+; COEXEC-NEXT: global_store_b128 v32, v[28:31], s[8:9] offset:16
+; COEXEC-NEXT: global_store_b128 v32, v[24:27], s[8:9]
+; COEXEC-NEXT: global_store_b128 v32, v[20:23], s[8:9] offset:144
+; COEXEC-NEXT: global_store_b128 v32, v[16:19], s[8:9] offset:128
+; COEXEC-NEXT: global_store_b128 v32, v[12:15], s[8:9] offset:272
+; COEXEC-NEXT: global_store_b128 v32, v[8:11], s[8:9] offset:256
+; COEXEC-NEXT: global_store_b128 v32, v[4:7], s[8:9] offset:400
+; COEXEC-NEXT: global_store_b128 v32, v[0:3], s[8:9] offset:384
; COEXEC-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; COEXEC-NEXT: s_endpgm
;
@@ -93,8 +91,8 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_clause 0x1
-; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
-; GCN-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
+; GCN-NEXT: s_load_b128 s[8:11], s[4:5], 0x8 nv
+; GCN-NEXT: s_load_b32 s1, s[4:5], 0x0 nv
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; GCN-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
@@ -104,7 +102,7 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; GCN-NEXT: v_dual_mov_b32 v11, v0 :: v_dual_mov_b32 v12, v0
; GCN-NEXT: v_dual_mov_b32 v13, v0 :: v_dual_mov_b32 v14, v0
; GCN-NEXT: s_wait_kmcnt 0x0
-; GCN-NEXT: s_bitcmp1_b32 s0, 0
+; GCN-NEXT: s_bitcmp1_b32 s10, 0
; GCN-NEXT: v_dual_mov_b32 v15, v0 :: v_dual_mov_b32 v16, v0
; GCN-NEXT: s_cselect_b32 s0, -1, 0
; GCN-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v0
@@ -124,9 +122,9 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; GCN-NEXT: v_nop
; GCN-NEXT: v_nop
; GCN-NEXT: v_nop
-; GCN-NEXT: v_mov_b32_e32 v92, s2
+; GCN-NEXT: v_mov_b32_e32 v92, s1
; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0
-; GCN-NEXT: s_add_co_i32 s2, s2, s1
+; GCN-NEXT: s_add_co_i32 s1, s1, s11
; GCN-NEXT: ds_load_tr16_b128 v[32:35], v92
; GCN-NEXT: ds_load_tr16_b128 v[36:39], v92 offset:64
; GCN-NEXT: ds_load_tr16_b128 v[40:43], v92 offset:128
@@ -157,19 +155,17 @@ define amdgpu_kernel void @ds_wmma(ptr addrspace(3) %base, ptr addrspace(1) %out
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
; GCN-NEXT: s_cbranch_vccnz .LBB0_1
; GCN-NEXT: ; %bb.2: ; %end
-; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GCN-NEXT: v_nop
; GCN-NEXT: v_mov_b32_e32 v32, 0
-; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_clause 0x7
-; GCN-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
-; GCN-NEXT: global_store_b128 v32, v[24:27], s[0:1]
-; GCN-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
-; GCN-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
-; GCN-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
-; GCN-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
-; GCN-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
-; GCN-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
+; GCN-NEXT: global_store_b128 v32, v[28:31], s[8:9] offset:16
+; GCN-NEXT: global_store_b128 v32, v[24:27], s[8:9]
+; GCN-NEXT: global_store_b128 v32, v[20:23], s[8:9] offset:144
+; GCN-NEXT: global_store_b128 v32, v[16:19], s[8:9] offset:128
+; GCN-NEXT: global_store_b128 v32, v[12:15], s[8:9] offset:272
+; GCN-NEXT: global_store_b128 v32, v[8:11], s[8:9] offset:256
+; GCN-NEXT: global_store_b128 v32, v[4:7], s[8:9] offset:400
+; GCN-NEXT: global_store_b128 v32, v[0:3], s[8:9] offset:384
; GCN-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GCN-NEXT: s_endpgm
entry:
@@ -248,9 +244,9 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-LABEL: ds_wmma_permute:
; COEXEC: ; %bb.0: ; %entry
; COEXEC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; COEXEC-NEXT: s_mov_b32 s6, 0
+; COEXEC-NEXT: s_mov_b32 s2, 0
; COEXEC-NEXT: s_clause 0x1
-; COEXEC-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
+; COEXEC-NEXT: s_load_b128 s[8:11], s[4:5], 0x0 nv
; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
; COEXEC-NEXT: v_mov_b32_e32 v0, 0
; COEXEC-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
@@ -278,16 +274,16 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: v_cmp_ne_u32_e64 s0, 1, v32
; COEXEC-NEXT: .LBB1_1: ; %loop
; COEXEC-NEXT: ; =>This Inner Loop Header: Depth=1
-; COEXEC-NEXT: s_add_co_i32 s7, s2, s6
-; COEXEC-NEXT: s_add_co_i32 s8, s3, s6
-; COEXEC-NEXT: s_add_co_i32 s6, s6, s1
+; COEXEC-NEXT: s_add_co_i32 s3, s8, s2
+; COEXEC-NEXT: s_add_co_i32 s4, s9, s2
+; COEXEC-NEXT: s_add_co_i32 s2, s2, s1
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
; COEXEC-NEXT: v_nop
-; COEXEC-NEXT: v_mov_b32_e32 v124, s7
+; COEXEC-NEXT: v_mov_b32_e32 v124, s3
; COEXEC-NEXT: s_and_b32 vcc_lo, exec_lo, s0
-; COEXEC-NEXT: v_mov_b32_e32 v156, s8
+; COEXEC-NEXT: v_mov_b32_e32 v156, s4
; COEXEC-NEXT: ds_load_tr16_b128 v[32:35], v124
; COEXEC-NEXT: ds_load_tr16_b128 v[36:39], v124 offset:64
; COEXEC-NEXT: ds_load_tr16_b128 v[40:43], v156
@@ -347,17 +343,15 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; COEXEC-NEXT: s_cbranch_vccnz .LBB1_1
; COEXEC-NEXT: ; %bb.2: ; %end
; COEXEC-NEXT: v_mov_b32_e32 v32, 0
-; COEXEC-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
-; COEXEC-NEXT: s_wait_kmcnt 0x0
; COEXEC-NEXT: s_clause 0x7
-; COEXEC-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
-; COEXEC-NEXT: global_store_b128 v32, v[24:27], s[0:1]
-; COEXEC-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
-; COEXEC-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
-; COEXEC-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
-; COEXEC-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
-; COEXEC-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
-; COEXEC-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
+; COEXEC-NEXT: global_store_b128 v32, v[28:31], s[10:11] offset:16
+; COEXEC-NEXT: global_store_b128 v32, v[24:27], s[10:11]
+; COEXEC-NEXT: global_store_b128 v32, v[20:23], s[10:11] offset:144
+; COEXEC-NEXT: global_store_b128 v32, v[16:19], s[10:11] offset:128
+; COEXEC-NEXT: global_store_b128 v32, v[12:15], s[10:11] offset:272
+; COEXEC-NEXT: global_store_b128 v32, v[8:11], s[10:11] offset:256
+; COEXEC-NEXT: global_store_b128 v32, v[4:7], s[10:11] offset:400
+; COEXEC-NEXT: global_store_b128 v32, v[0:3], s[10:11] offset:384
; COEXEC-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; COEXEC-NEXT: s_endpgm
;
@@ -366,9 +360,9 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; GCN-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GCN-NEXT: s_clause 0x1
; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
-; GCN-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
+; GCN-NEXT: s_load_b128 s[8:11], s[4:5], 0x0 nv
; GCN-NEXT: v_mov_b32_e32 v0, 0
-; GCN-NEXT: s_mov_b32 s6, 0
+; GCN-NEXT: s_mov_b32 s2, 0
; GCN-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
; GCN-NEXT: v_dual_mov_b32 v3, v0 :: v_dual_mov_b32 v4, v0
; GCN-NEXT: v_dual_mov_b32 v5, v0 :: v_dual_mov_b32 v6, v0
@@ -393,11 +387,11 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; GCN-NEXT: v_dual_mov_b32 v30, v0 :: v_dual_mov_b32 v31, v0
; GCN-NEXT: .LBB1_1: ; %loop
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: s_add_co_i32 s7, s2, s6
-; GCN-NEXT: s_add_co_i32 s8, s3, s6
-; GCN-NEXT: v_dual_mov_b32 v96, s7 :: v_dual_mov_b32 v97, s8
+; GCN-NEXT: s_add_co_i32 s3, s8, s2
+; GCN-NEXT: s_add_co_i32 s4, s9, s2
+; GCN-NEXT: v_dual_mov_b32 v96, s3 :: v_dual_mov_b32 v97, s4
; GCN-NEXT: s_and_b32 vcc_lo, exec_lo, s0
-; GCN-NEXT: s_add_co_i32 s6, s6, s1
+; GCN-NEXT: s_add_co_i32 s2, s2, s1
; GCN-NEXT: ds_load_tr16_b128 v[32:35], v96
; GCN-NEXT: ds_load_tr16_b128 v[36:39], v96 offset:64
; GCN-NEXT: ds_load_tr16_b128 v[40:43], v97
@@ -456,19 +450,17 @@ define amdgpu_kernel void @ds_wmma_permute(ptr addrspace(3) %base, ptr addrspace
; GCN-NEXT: v_wmma_f32_16x16x32_f16 v[0:7], v[80:87], v[88:95], v[0:7]
; GCN-NEXT: s_cbranch_vccnz .LBB1_1
; GCN-NEXT: ; %bb.2: ; %end
-; GCN-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GCN-NEXT: v_nop
; GCN-NEXT: v_mov_b32_e32 v32, 0
-; GCN-NEXT: s_wait_kmcnt 0x0
; GCN-NEXT: s_clause 0x7
-; GCN-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:16
-; GCN-NEXT: global_store_b128 v32, v[24:27], s[0:1]
-; GCN-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:144
-; GCN-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:128
-; GCN-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:272
-; GCN-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:256
-; GCN-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:400
-; GCN-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:384
+; GCN-NEXT: global_store_b128 v32, v[28:31], s[10:11] offset:16
+; GCN-NEXT: global_store_b128 v32, v[24:27], s[10:11]
+; GCN-NEXT: global_store_b128 v32, v[20:23], s[10:11] offset:144
+; GCN-NEXT: global_store_b128 v32, v[16:19], s[10:11] offset:128
+; GCN-NEXT: global_store_b128 v32, v[12:15], s[10:11] offset:272
+; GCN-NEXT: global_store_b128 v32, v[8:11], s[10:11] offset:256
+; GCN-NEXT: global_store_b128 v32, v[4:7], s[10:11] offset:400
+; GCN-NEXT: global_store_b128 v32, v[0:3], s[10:11] offset:384
; GCN-NEXT: s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
; GCN-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll b/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll
index 4cbd41c1b1965..8e4bea2eff937 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-cond-add-sub.ll
@@ -127,25 +127,24 @@ bb:
define amdgpu_kernel void @add_adde(ptr addrspace(1) nocapture %arg, i32 %a) {
; GCN-LABEL: add_adde:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_load_dword s4, s[4:5], 0xb
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
-; GCN-NEXT: v_mov_b32_e32 v5, s4
+; GCN-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GCN-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64
+; GCN-NEXT: v_mov_b32_e32 v5, s2
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_addc_u32_e32 v0, vcc, v5, v4, vcc
-; GCN-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: add_adde:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -171,25 +170,24 @@ bb:
define amdgpu_kernel void @adde_add(ptr addrspace(1) nocapture %arg, i32 %a) {
; GCN-LABEL: adde_add:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_load_dword s4, s[4:5], 0xb
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
-; GCN-NEXT: v_mov_b32_e32 v5, s4
+; GCN-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GCN-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64
+; GCN-NEXT: v_mov_b32_e32 v5, s2
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_addc_u32_e32 v0, vcc, v4, v5, vcc
-; GCN-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: adde_add:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -215,25 +213,24 @@ bb:
define amdgpu_kernel void @sub_sube(ptr addrspace(1) nocapture %arg, i32 %a) {
; GCN-LABEL: sub_sube:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_load_dword s4, s[4:5], 0xb
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
-; GCN-NEXT: v_mov_b32_e32 v5, s4
+; GCN-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GCN-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64
+; GCN-NEXT: v_mov_b32_e32 v5, s2
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_subb_u32_e32 v0, vcc, v4, v5, vcc
-; GCN-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: sub_sube:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -259,26 +256,25 @@ bb:
define amdgpu_kernel void @sub_sube_commuted(ptr addrspace(1) nocapture %arg, i32 %a) {
; GCN-LABEL: sub_sube_commuted:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_load_dword s4, s[4:5], 0xb
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GCN-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_subbrev_u32_e32 v0, vcc, 0, v4, vcc
-; GCN-NEXT: v_sub_i32_e32 v0, vcc, s4, v0
+; GCN-NEXT: v_sub_i32_e32 v0, vcc, s2, v0
; GCN-NEXT: v_add_i32_e32 v0, vcc, 0x64, v0
-; GCN-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: sub_sube_commuted:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -306,25 +302,24 @@ bb:
define amdgpu_kernel void @sube_sub(ptr addrspace(1) nocapture %arg, i32 %a) {
; GCN-LABEL: sube_sub:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_load_dword s4, s[4:5], 0xb
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
-; GCN-NEXT: v_mov_b32_e32 v5, s4
+; GCN-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GCN-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64
+; GCN-NEXT: v_mov_b32_e32 v5, s2
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_subb_u32_e32 v0, vcc, v4, v5, vcc
-; GCN-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: sube_sub:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -350,25 +345,24 @@ bb:
define amdgpu_kernel void @zext_flclass(ptr addrspace(1) nocapture %arg, float %x) {
; GCN-LABEL: zext_flclass:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_load_dword s4, s[4:5], 0xb
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; GCN-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GCN-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
; GCN-NEXT: v_mov_b32_e32 v3, 0x260
-; GCN-NEXT: v_cmp_class_f32_e32 vcc, s4, v3
+; GCN-NEXT: v_cmp_class_f32_e32 vcc, s2, v3
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_addc_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; GCN-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: zext_flclass:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v2, 0x260
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -392,25 +386,24 @@ bb:
define amdgpu_kernel void @sext_flclass(ptr addrspace(1) nocapture %arg, float %x) {
; GCN-LABEL: sext_flclass:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_load_dword s4, s[4:5], 0xb
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
+; GCN-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GCN-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
; GCN-NEXT: v_mov_b32_e32 v3, 0x260
-; GCN-NEXT: v_cmp_class_f32_e32 vcc, s4, v3
+; GCN-NEXT: v_cmp_class_f32_e32 vcc, s2, v3
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_subbrev_u32_e32 v2, vcc, 0, v2, vcc
-; GCN-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; GCN-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: sext_flclass:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v2, 0x260
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -557,25 +550,24 @@ bb:
define amdgpu_kernel void @sub_addcarry(ptr addrspace(1) nocapture %arg, i32 %a) {
; GCN-LABEL: sub_addcarry:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_load_dword s4, s[4:5], 0xb
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GCN-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v4, vcc
-; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s4, v0
-; GCN-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: sub_addcarry:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -601,25 +593,24 @@ bb:
define amdgpu_kernel void @sub_subcarry(ptr addrspace(1) nocapture %arg, i32 %a) {
; GCN-LABEL: sub_subcarry:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_load_dword s4, s[4:5], 0xb
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
-; GCN-NEXT: v_mov_b32_e32 v5, s4
+; GCN-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GCN-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64
+; GCN-NEXT: v_mov_b32_e32 v5, s2
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_subb_u32_e32 v0, vcc, v4, v5, vcc
-; GCN-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: sub_subcarry:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -647,20 +638,20 @@ define amdgpu_kernel void @sub_zext_setcc_commute(ptr addrspace(1) nocapture %ar
; GCN-LABEL: sub_zext_setcc_commute:
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
; GCN-NEXT: v_mov_b32_e32 v3, 0
-; GCN-NEXT: buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GCN-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s4, v0
-; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s5, v0
-; GCN-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s3, v0
+; GCN-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: sub_zext_setcc_commute:
@@ -696,20 +687,20 @@ define amdgpu_kernel void @sub_sext_setcc_commute(ptr addrspace(1) nocapture %ar
; GCN-LABEL: sub_sext_setcc_commute:
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, 0
; GCN-NEXT: v_lshlrev_b32_e32 v2, 2, v0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, 0
; GCN-NEXT: v_mov_b32_e32 v3, 0
-; GCN-NEXT: buffer_load_dword v4, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b64 s[4:5], s[0:1]
+; GCN-NEXT: buffer_load_dword v4, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: v_cmp_gt_u32_e32 vcc, v0, v1
; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_sub_i32_e32 v0, vcc, v0, v4
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s4, v0
-; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s5, v0
-; GCN-NEXT: buffer_store_dword v0, v[2:3], s[0:3], 0 addr64
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT: v_subrev_i32_e32 v0, vcc, s3, v0
+; GCN-NEXT: buffer_store_dword v0, v[2:3], s[4:7], 0 addr64
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: sub_sext_setcc_commute:
diff --git a/llvm/test/CodeGen/AMDGPU/combine-reg-or-const.ll b/llvm/test/CodeGen/AMDGPU/combine-reg-or-const.ll
index 83e62cbb9b6fb..4ad5b53c25d42 100644
--- a/llvm/test/CodeGen/AMDGPU/combine-reg-or-const.ll
+++ b/llvm/test/CodeGen/AMDGPU/combine-reg-or-const.ll
@@ -8,25 +8,23 @@ define protected amdgpu_kernel void @_Z11test_kernelPii(ptr addrspace(1) nocaptu
; CHECK-NEXT: s_mov_b32 flat_scratch_lo, s13
; CHECK-NEXT: s_add_i32 s12, s12, s17
; CHECK-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CHECK-NEXT: s_load_dword s0, s[8:9], 0x2
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_cmp_lg_u32 s0, 3
+; CHECK-NEXT: s_cmp_lg_u32 s2, 3
; CHECK-NEXT: s_cbranch_scc1 .LBB0_2
; CHECK-NEXT: ; %bb.1: ; %if.then
-; CHECK-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0
-; CHECK-NEXT: s_and_b32 s4, s0, 0xffff
-; CHECK-NEXT: s_mov_b32 s1, 0
+; CHECK-NEXT: s_and_b32 s4, s2, 0xffff
+; CHECK-NEXT: s_mov_b32 s3, 0
; CHECK-NEXT: s_mul_i32 s6, s4, 0xaaab
-; CHECK-NEXT: s_lshl_b64 s[4:5], s[0:1], 2
-; CHECK-NEXT: s_lshr_b32 s1, s6, 19
-; CHECK-NEXT: s_mul_i32 s1, s1, 12
-; CHECK-NEXT: s_sub_i32 s6, s0, s1
-; CHECK-NEXT: s_and_b32 s7, s6, 0xffff
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_add_u32 s0, s2, s4
-; CHECK-NEXT: s_addc_u32 s1, s3, s5
-; CHECK-NEXT: s_bfe_u32 s2, s6, 0xd0003
-; CHECK-NEXT: s_add_i32 s2, s2, s7
+; CHECK-NEXT: s_lshl_b64 s[4:5], s[2:3], 2
+; CHECK-NEXT: s_lshr_b32 s3, s6, 19
+; CHECK-NEXT: s_mul_i32 s3, s3, 12
+; CHECK-NEXT: s_sub_i32 s2, s2, s3
+; CHECK-NEXT: s_and_b32 s3, s2, 0xffff
+; CHECK-NEXT: s_add_u32 s0, s0, s4
+; CHECK-NEXT: s_addc_u32 s1, s1, s5
+; CHECK-NEXT: s_bfe_u32 s2, s2, 0xd0003
+; CHECK-NEXT: s_add_i32 s2, s2, s3
; CHECK-NEXT: s_addk_i32 s2, 0xc0
; CHECK-NEXT: v_mov_b32_e32 v0, s0
; CHECK-NEXT: v_mov_b32_e32 v1, s1
diff --git a/llvm/test/CodeGen/AMDGPU/constant-fold-mi-operands.ll b/llvm/test/CodeGen/AMDGPU/constant-fold-mi-operands.ll
index 0691ae53d3bc5..30511a9c8ea26 100644
--- a/llvm/test/CodeGen/AMDGPU/constant-fold-mi-operands.ll
+++ b/llvm/test/CodeGen/AMDGPU/constant-fold-mi-operands.ll
@@ -37,9 +37,8 @@ define amdgpu_kernel void @fold_mi_v_or_0(ptr addrspace(1) %out) {
}
; GCN-LABEL: {{^}}fold_mi_s_or_0:
-; GCN: s_load_dword [[SVAL:s[0-9]+]]
-; GCN-NOT: [[SVAL]]
-; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL]]
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL:s[0-9]+]]
; GCN-NOT: [[VVAL]]
; GCN: buffer_store_dword [[VVAL]]
define amdgpu_kernel void @fold_mi_s_or_0(ptr addrspace(1) %out, i32 %x) #0 {
@@ -62,9 +61,8 @@ define amdgpu_kernel void @fold_mi_v_xor_0(ptr addrspace(1) %out) {
}
; GCN-LABEL: {{^}}fold_mi_s_xor_0:
-; GCN: s_load_dword [[SVAL:s[0-9]+]]
-; GCN-NOT: [[SVAL]]
-; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL]]
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN: v_mov_b32_e32 [[VVAL:v[0-9]+]], [[SVAL:s[0-9]+]]
; GCN-NOT: [[VVAL]]
; GCN: buffer_store_dword [[VVAL]]
define amdgpu_kernel void @fold_mi_s_xor_0(ptr addrspace(1) %out, i32 %x) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz.ll b/llvm/test/CodeGen/AMDGPU/ctlz.ll
index 8c3b5be0b0ae0..b561b6033949f 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz.ll
@@ -24,10 +24,9 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
define amdgpu_kernel void @s_ctlz_i32(ptr addrspace(1) noalias %out, i32 %val) nounwind {
; SI-LABEL: s_ctlz_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_flbit_i32_b32 s2, s2
; SI-NEXT: s_min_u32 s4, s2, 32
; SI-NEXT: s_mov_b32 s2, -1
@@ -37,15 +36,16 @@ define amdgpu_kernel void @s_ctlz_i32(ptr addrspace(1) noalias %out, i32 %val) n
;
; VI-LABEL: s_ctlz_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_flbit_i32_b32 s4, s6
-; VI-NEXT: s_min_u32 s4, s4, 32
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_flbit_i32_b32 s0, s2
+; VI-NEXT: s_min_u32 s0, s0, 32
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; EG-LABEL: s_ctlz_i32:
@@ -62,9 +62,7 @@ define amdgpu_kernel void @s_ctlz_i32(ptr addrspace(1) noalias %out, i32 %val) n
;
; GFX10-LABEL: s_ctlz_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_flbit_i32_b32 s2, s2
@@ -88,9 +86,7 @@ define amdgpu_kernel void @s_ctlz_i32(ptr addrspace(1) noalias %out, i32 %val) n
;
; GFX11-LABEL: s_ctlz_i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_clz_i32_u32 s2, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
index 867bd8ee26227..53ec1380ea766 100644
--- a/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctlz_zero_poison.ll
@@ -29,10 +29,9 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
define amdgpu_kernel void @s_ctlz_zero_poison_i32(ptr addrspace(1) noalias %out, i32 %val) nounwind {
; SI-LABEL: s_ctlz_zero_poison_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_flbit_i32_b32 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -41,13 +40,12 @@ define amdgpu_kernel void @s_ctlz_zero_poison_i32(ptr addrspace(1) noalias %out,
;
; VI-LABEL: s_ctlz_zero_poison_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_flbit_i32_b32 s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_flbit_i32_b32 s0, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -318,10 +316,9 @@ define amdgpu_kernel void @v_ctlz_zero_poison_v4i32(ptr addrspace(1) noalias %ou
define amdgpu_kernel void @s_ctlz_zero_poison_i8_with_select(ptr addrspace(1) noalias %out, i8 %val) nounwind {
; SI-LABEL: s_ctlz_zero_poison_i8_with_select:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_lshl_b32 s2, s2, 24
; SI-NEXT: s_flbit_i32_b32 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
@@ -331,14 +328,13 @@ define amdgpu_kernel void @s_ctlz_zero_poison_i8_with_select(ptr addrspace(1) no
;
; VI-LABEL: s_ctlz_zero_poison_i8_with_select:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b32 s2, s2, 24
-; VI-NEXT: s_flbit_i32_b32 s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_lshl_b32 s0, s2, 24
+; VI-NEXT: s_flbit_i32_b32 s0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_byte v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -392,10 +388,9 @@ define amdgpu_kernel void @s_ctlz_zero_poison_i8_with_select(ptr addrspace(1) no
define amdgpu_kernel void @s_ctlz_zero_poison_i16_with_select(ptr addrspace(1) noalias %out, i16 %val) nounwind {
; SI-LABEL: s_ctlz_zero_poison_i16_with_select:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_lshl_b32 s2, s2, 16
; SI-NEXT: s_flbit_i32_b32 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
@@ -405,14 +400,13 @@ define amdgpu_kernel void @s_ctlz_zero_poison_i16_with_select(ptr addrspace(1) n
;
; VI-LABEL: s_ctlz_zero_poison_i16_with_select:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b32 s2, s2, 16
-; VI-NEXT: s_flbit_i32_b32 s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_lshl_b32 s0, s2, 16
+; VI-NEXT: s_flbit_i32_b32 s0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -466,10 +460,9 @@ define amdgpu_kernel void @s_ctlz_zero_poison_i16_with_select(ptr addrspace(1) n
define amdgpu_kernel void @s_ctlz_zero_poison_i32_with_select(ptr addrspace(1) noalias %out, i32 %val) nounwind {
; SI-LABEL: s_ctlz_zero_poison_i32_with_select:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_flbit_i32_b32 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -478,13 +471,12 @@ define amdgpu_kernel void @s_ctlz_zero_poison_i32_with_select(ptr addrspace(1) n
;
; VI-LABEL: s_ctlz_zero_poison_i32_with_select:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_flbit_i32_b32 s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_flbit_i32_b32 s0, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -2225,25 +2217,22 @@ define i7 @v_ctlz_zero_poison_i7(i7 %val) {
define amdgpu_kernel void @s_ctlz_zero_poison_i18(ptr addrspace(1) noalias %out, i18 %val) nounwind {
; SI-LABEL: s_ctlz_zero_poison_i18:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_lshl_b32 s2, s2, 14
; SI-NEXT: s_flbit_i32_b32 s4, s2
+; SI-NEXT: s_bfe_u32 s5, s4, 0x20010
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: s_bfe_u32 s4, s4, 0x20010
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: buffer_store_byte v1, off, s[0:3], 0 offset:2
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0 offset:2
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_ctlz_zero_poison_i18:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshl_b32 s2, s2, 14
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -2252,8 +2241,8 @@ define amdgpu_kernel void @s_ctlz_zero_poison_i18(ptr addrspace(1) noalias %out,
; VI-NEXT: s_add_u32 s0, s0, 2
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_bfe_u32 s2, s2, 0x20010
+; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_mov_b32_e32 v2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop.ll b/llvm/test/CodeGen/AMDGPU/ctpop.ll
index 2c210a63533b1..9986928259d04 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop.ll
@@ -17,26 +17,28 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
define amdgpu_kernel void @s_ctpop_i32(ptr addrspace(1) noalias %out, i32 %val) nounwind {
; SI-LABEL: s_ctpop_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_bcnt1_i32_b32 s4, s6
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_bcnt1_i32_b32 s0, s2
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_ctpop_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bcnt1_i32_b32 s4, s6
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_bcnt1_i32_b32 s0, s2
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; EG-LABEL: s_ctpop_i32:
@@ -1640,53 +1642,52 @@ define amdgpu_kernel void @ctpop_i32_in_br(ptr addrspace(1) %out, ptr addrspace(
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dword s6, s[4:5], 0x16
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dword s5, s[4:5], 0xd
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s6, 0
; SI-NEXT: s_cbranch_scc0 .LBB14_4
; SI-NEXT: ; %bb.1: ; %else
-; SI-NEXT: s_load_dword s6, s[2:3], 0x1
+; SI-NEXT: s_load_dword s4, s[2:3], 0x1
; SI-NEXT: s_mov_b64 s[2:3], 0
; SI-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b64 vcc, vcc
; SI-NEXT: s_cbranch_vccnz .LBB14_3
; SI-NEXT: .LBB14_2: ; %if
-; SI-NEXT: s_load_dword s2, s[4:5], 0xd
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_bcnt1_i32_b32 s6, s2
+; SI-NEXT: s_bcnt1_i32_b32 s4, s5
; SI-NEXT: .LBB14_3: ; %endif
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s6
+; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
; SI-NEXT: .LBB14_4:
-; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: ; implicit-def: $sgpr4
; SI-NEXT: s_branch .LBB14_2
;
; VI-LABEL: ctpop_i32_in_br:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dword s6, s[4:5], 0x58
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dword s5, s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s6, 0
; VI-NEXT: s_cbranch_scc0 .LBB14_4
; VI-NEXT: ; %bb.1: ; %else
-; VI-NEXT: s_load_dword s6, s[2:3], 0x4
+; VI-NEXT: s_load_dword s4, s[2:3], 0x4
; VI-NEXT: s_cbranch_execnz .LBB14_3
; VI-NEXT: .LBB14_2: ; %if
-; VI-NEXT: s_load_dword s2, s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bcnt1_i32_b32 s6, s2
+; VI-NEXT: s_bcnt1_i32_b32 s4, s5
; VI-NEXT: .LBB14_3: ; %endif
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_endpgm
; VI-NEXT: .LBB14_4:
-; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: ; implicit-def: $sgpr4
; VI-NEXT: s_branch .LBB14_2
;
; EG-LABEL: ctpop_i32_in_br:
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop16.ll b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
index 3b3f59363e252..a5a9f4f6b2191 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop16.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop16.ll
@@ -16,28 +16,30 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
define amdgpu_kernel void @s_ctpop_i16(ptr addrspace(1) noalias %out, i16 %val) nounwind {
; SI-LABEL: s_ctpop_i16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0xffff
-; SI-NEXT: s_bcnt1_i32_b32 s4, s4
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0xffff
+; SI-NEXT: s_bcnt1_i32_b32 s0, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_ctpop_i16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s4, s6, 0xffff
-; VI-NEXT: s_bcnt1_i32_b32 s4, s4
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_and_b32 s0, s2, 0xffff
+; VI-NEXT: s_bcnt1_i32_b32 s0, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; EG-LABEL: s_ctpop_i16:
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop64.ll b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
index d3d40e4c3c9db..936cdddf1532b 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop64.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
@@ -455,85 +455,79 @@ define amdgpu_kernel void @v_ctpop_v4i64(ptr addrspace(1) noalias %out, ptr addr
define amdgpu_kernel void @ctpop_i64_in_br(ptr addrspace(1) %out, ptr addrspace(1) %in, i64 %ctpop_arg, i32 %cond) {
; SI-LABEL: ctpop_i64_in_br:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s8, s[4:5], 0xf
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_lg_u32 s8, 0
+; SI-NEXT: s_cmp_lg_u32 s6, 0
; SI-NEXT: s_cbranch_scc0 .LBB7_4
; SI-NEXT: ; %bb.1: ; %else
-; SI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x2
+; SI-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x2
; SI-NEXT: s_mov_b64 s[2:3], 0
; SI-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b64 vcc, vcc
; SI-NEXT: s_cbranch_vccnz .LBB7_3
; SI-NEXT: .LBB7_2: ; %if
-; SI-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
-; SI-NEXT: s_mov_b32 s5, 0
+; SI-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; SI-NEXT: s_mov_b32 s7, 0
; SI-NEXT: .LBB7_3: ; %endif
-; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v0, s6
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v1, s7
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
; SI-NEXT: .LBB7_4:
-; SI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; SI-NEXT: ; implicit-def: $sgpr6_sgpr7
; SI-NEXT: s_branch .LBB7_2
;
; VI-LABEL: ctpop_i64_in_br:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s8, s[4:5], 0x3c
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_lg_u32 s8, 0
+; VI-NEXT: s_cmp_lg_u32 s6, 0
; VI-NEXT: s_cbranch_scc0 .LBB7_4
; VI-NEXT: ; %bb.1: ; %else
-; VI-NEXT: s_load_dwordx2 s[4:5], s[2:3], 0x8
+; VI-NEXT: s_load_dwordx2 s[6:7], s[2:3], 0x8
; VI-NEXT: s_cbranch_execnz .LBB7_3
; VI-NEXT: .LBB7_2: ; %if
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bcnt1_i32_b64 s4, s[6:7]
-; VI-NEXT: s_mov_b32 s5, 0
+; VI-NEXT: s_bcnt1_i32_b64 s6, s[4:5]
+; VI-NEXT: s_mov_b32 s7, 0
; VI-NEXT: .LBB7_3: ; %endif
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v0, s6
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v1, s7
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
; VI-NEXT: .LBB7_4:
-; VI-NEXT: ; implicit-def: $sgpr4_sgpr5
+; VI-NEXT: ; implicit-def: $sgpr6_sgpr7
; VI-NEXT: s_branch .LBB7_2
;
; GFX12-LABEL: ctpop_i64_in_br:
; GFX12: ; %bb.0: ; %entry
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x3c
+; GFX12-NEXT: s_load_b96 s[8:10], s[4:5], 0x34
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_mov_b32 s4, 1
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_cmp_lg_u32 s6, 0
-; GFX12-NEXT: s_mov_b32 s6, 1
+; GFX12-NEXT: s_cmp_lg_u32 s10, 0
; GFX12-NEXT: s_cbranch_scc0 .LBB7_2
; GFX12-NEXT: ; %bb.1: ; %else
; GFX12-NEXT: s_load_b64 s[2:3], s[2:3], 0x8
-; GFX12-NEXT: s_mov_b32 s6, 0
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: s_branch .LBB7_3
; GFX12-NEXT: .LBB7_2:
; GFX12-NEXT: ; implicit-def: $sgpr2_sgpr3
; GFX12-NEXT: .LBB7_3: ; %Flow
-; GFX12-NEXT: s_xor_b32 s6, s6, 1
+; GFX12-NEXT: s_xor_b32 s4, s4, 1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: s_cmp_lg_u32 s6, 0
+; GFX12-NEXT: s_cmp_lg_u32 s4, 0
; GFX12-NEXT: s_cbranch_scc1 .LBB7_5
; GFX12-NEXT: ; %bb.4: ; %if
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x34
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX12-NEXT: s_bcnt1_i32_b64 s2, s[8:9]
; GFX12-NEXT: s_mov_b32 s3, 0
; GFX12-NEXT: .LBB7_5: ; %endif
; GFX12-NEXT: s_wait_kmcnt 0x0
@@ -563,28 +557,32 @@ endif:
define amdgpu_kernel void @s_ctpop_i128(ptr addrspace(1) noalias %out, i128 %val) nounwind {
; SI-LABEL: s_ctpop_i128:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; SI-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; SI-NEXT: s_add_i32 s0, s0, s2
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
+; SI-NEXT: s_bcnt1_i32_b64 s1, s[2:3]
+; SI-NEXT: s_add_i32 s0, s1, s0
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_ctpop_i128:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; VI-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; VI-NEXT: s_add_i32 s0, s0, s2
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
+; VI-NEXT: s_bcnt1_i32_b64 s1, s[2:3]
+; VI-NEXT: s_add_i32 s0, s1, s0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
@@ -592,16 +590,16 @@ define amdgpu_kernel void @s_ctpop_i128(ptr addrspace(1) noalias %out, i128 %val
; GFX12-LABEL: s_ctpop_i128:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT: v_mov_b32_e32 v1, 0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX12-NEXT: s_bcnt1_i32_b64 s1, s[2:3]
+; GFX12-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX12-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_add_co_i32 s0, s1, s0
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: global_store_b32 v1, v0, s[4:5]
+; GFX12-NEXT: s_add_co_i32 s2, s3, s2
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-NEXT: s_endpgm
%ctpop = call i128 @llvm.ctpop.i128(i128 %val) nounwind readnone
%truncctpop = trunc i128 %ctpop to i32
diff --git a/llvm/test/CodeGen/AMDGPU/cttz.ll b/llvm/test/CodeGen/AMDGPU/cttz.ll
index 974e7179988e5..d068b19833829 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz.ll
@@ -22,10 +22,9 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
define amdgpu_kernel void @s_cttz_i32(ptr addrspace(1) noalias %out, i32 %val) nounwind {
; SI-LABEL: s_cttz_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_ff1_i32_b32 s2, s2
; SI-NEXT: s_min_u32 s4, s2, 32
; SI-NEXT: s_mov_b32 s2, -1
@@ -35,15 +34,16 @@ define amdgpu_kernel void @s_cttz_i32(ptr addrspace(1) noalias %out, i32 %val) n
;
; VI-LABEL: s_cttz_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ff1_i32_b32 s4, s6
-; VI-NEXT: s_min_u32 s4, s4, 32
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_ff1_i32_b32 s0, s2
+; VI-NEXT: s_min_u32 s0, s0, 32
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; EG-LABEL: s_cttz_i32:
@@ -60,9 +60,7 @@ define amdgpu_kernel void @s_cttz_i32(ptr addrspace(1) noalias %out, i32 %val) n
;
; GFX10-LABEL: s_cttz_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_ff1_i32_b32 s2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/cttz_zero_poison.ll b/llvm/test/CodeGen/AMDGPU/cttz_zero_poison.ll
index 60d87fe22f208..3a1164e11ab6f 100644
--- a/llvm/test/CodeGen/AMDGPU/cttz_zero_poison.ll
+++ b/llvm/test/CodeGen/AMDGPU/cttz_zero_poison.ll
@@ -16,10 +16,9 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
define amdgpu_kernel void @s_cttz_zero_poison_i32(ptr addrspace(1) noalias %out, i32 %val) nounwind {
; SI-LABEL: s_cttz_zero_poison_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_ff1_i32_b32 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -28,13 +27,12 @@ define amdgpu_kernel void @s_cttz_zero_poison_i32(ptr addrspace(1) noalias %out,
;
; VI-LABEL: s_cttz_zero_poison_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ff1_i32_b32 s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_ff1_i32_b32 s0, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -305,10 +303,9 @@ define amdgpu_kernel void @v_cttz_zero_poison_v4i32(ptr addrspace(1) noalias %ou
define amdgpu_kernel void @s_cttz_zero_poison_i8_with_select(ptr addrspace(1) noalias %out, i8 %val) nounwind {
; SI-LABEL: s_cttz_zero_poison_i8_with_select:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_ff1_i32_b32 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -317,13 +314,12 @@ define amdgpu_kernel void @s_cttz_zero_poison_i8_with_select(ptr addrspace(1) no
;
; VI-LABEL: s_cttz_zero_poison_i8_with_select:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ff1_i32_b32 s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_ff1_i32_b32 s0, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_byte v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -374,10 +370,9 @@ define amdgpu_kernel void @s_cttz_zero_poison_i8_with_select(ptr addrspace(1) no
define amdgpu_kernel void @s_cttz_zero_poison_i16_with_select(ptr addrspace(1) noalias %out, i16 %val) nounwind {
; SI-LABEL: s_cttz_zero_poison_i16_with_select:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_ff1_i32_b32 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -386,13 +381,12 @@ define amdgpu_kernel void @s_cttz_zero_poison_i16_with_select(ptr addrspace(1) n
;
; VI-LABEL: s_cttz_zero_poison_i16_with_select:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ff1_i32_b32 s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_ff1_i32_b32 s0, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -443,10 +437,9 @@ define amdgpu_kernel void @s_cttz_zero_poison_i16_with_select(ptr addrspace(1) n
define amdgpu_kernel void @s_cttz_zero_poison_i32_with_select(ptr addrspace(1) noalias %out, i32 %val) nounwind {
; SI-LABEL: s_cttz_zero_poison_i32_with_select:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_ff1_i32_b32 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -455,13 +448,12 @@ define amdgpu_kernel void @s_cttz_zero_poison_i32_with_select(ptr addrspace(1) n
;
; VI-LABEL: s_cttz_zero_poison_i32_with_select:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ff1_i32_b32 s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_ff1_i32_b32 s0, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 0eed0ba50092b..a257a1e9c8b6b 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -2945,10 +2945,8 @@ define amdgpu_kernel void @cvt_f32_ubyte0_vector() local_unnamed_addr {
; SI-NEXT: buffer_load_ubyte v1, off, s[0:3], 0 offset:2
; SI-NEXT: buffer_load_ubyte v2, off, s[0:3], 0 offset:1
; SI-NEXT: buffer_load_ubyte v3, off, s[0:3], 0
-; SI-NEXT: s_load_dword s0, s[0:1], 0x0
; SI-NEXT: s_waitcnt vmcnt(3)
; SI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_fma_f32 v0, s0, v0, 0.5
; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: s_waitcnt vmcnt(2)
@@ -2970,10 +2968,8 @@ define amdgpu_kernel void @cvt_f32_ubyte0_vector() local_unnamed_addr {
; VI-NEXT: buffer_load_ubyte v1, off, s[0:3], 0 offset:2
; VI-NEXT: buffer_load_ubyte v2, off, s[0:3], 0 offset:1
; VI-NEXT: buffer_load_ubyte v3, off, s[0:3], 0
-; VI-NEXT: s_load_dword s0, s[0:1], 0x0
; VI-NEXT: s_waitcnt vmcnt(3)
; VI-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mul_f32_e32 v0, s0, v0
; VI-NEXT: v_add_f32_e32 v0, 0.5, v0
; VI-NEXT: v_cvt_i32_f32_e32 v0, v0
@@ -2996,11 +2992,8 @@ define amdgpu_kernel void @cvt_f32_ubyte0_vector() local_unnamed_addr {
; GFX10-NEXT: global_load_ubyte v2, v0, s[0:1] offset:2
; GFX10-NEXT: global_load_ubyte v3, v0, s[0:1] offset:1
; GFX10-NEXT: global_load_ubyte v4, v0, s[0:1]
-; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_load_dword s0, s[0:1], 0x0
; GFX10-NEXT: s_waitcnt vmcnt(3)
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_fma_f32 v0, s0, v0, 0.5
; GFX10-NEXT: v_cvt_i32_f32_e32 v0, v0
; GFX10-NEXT: s_waitcnt vmcnt(2)
@@ -3045,10 +3038,8 @@ define amdgpu_kernel void @cvt_f32_ubyte0_vector() local_unnamed_addr {
; GFX11-NEXT: global_load_u8 v2, v0, s[0:1] offset:2
; GFX11-NEXT: global_load_u8 v3, v0, s[0:1] offset:1
; GFX11-NEXT: global_load_u8 v0, v0, s[0:1]
-; GFX11-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX11-NEXT: s_waitcnt vmcnt(3)
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fma_f32 v1, s0, v1, 0.5
; GFX11-NEXT: v_cvt_i32_f32_e32 v1, v1
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_flr_i32_f32.ll b/llvm/test/CodeGen/AMDGPU/cvt_flr_i32_f32.ll
index 4c19928dc78fb..a79c649de5bb0 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_flr_i32_f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_flr_i32_f32.ll
@@ -10,13 +10,14 @@ declare float @llvm.floor.f32(float) #1
define amdgpu_kernel void @cvt_flr_i32_f32_0(ptr addrspace(1) %out, float %x) #0 {
; SI-SDAG-LABEL: cvt_flr_i32_f32_0:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cvt_flr_i32_f32_e32 v0, s6
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_cvt_flr_i32_f32_e32 v0, s2
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: cvt_flr_i32_f32_0:
@@ -32,9 +33,7 @@ define amdgpu_kernel void @cvt_flr_i32_f32_0(ptr addrspace(1) %out, float %x) #0
;
; GFX11-SDAG-LABEL: cvt_flr_i32_f32_0:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_floor_i32_f32_e32 v1, s2
@@ -60,14 +59,15 @@ define amdgpu_kernel void @cvt_flr_i32_f32_0(ptr addrspace(1) %out, float %x) #0
define amdgpu_kernel void @cvt_flr_i32_f32_1(ptr addrspace(1) %out, float %x) #0 {
; SI-SDAG-LABEL: cvt_flr_i32_f32_1:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_add_f32_e64 v0, s6, 1.0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_add_f32_e64 v0, s2, 1.0
; SI-SDAG-NEXT: v_cvt_flr_i32_f32_e32 v0, v0
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: cvt_flr_i32_f32_1:
@@ -84,9 +84,7 @@ define amdgpu_kernel void @cvt_flr_i32_f32_1(ptr addrspace(1) %out, float %x) #0
;
; GFX11-SDAG-LABEL: cvt_flr_i32_f32_1:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_add_f32_e64 v0, s2, 1.0
@@ -117,13 +115,14 @@ define amdgpu_kernel void @cvt_flr_i32_f32_1(ptr addrspace(1) %out, float %x) #0
define amdgpu_kernel void @cvt_flr_i32_f32_fabs(ptr addrspace(1) %out, float %x) #0 {
; SI-SDAG-LABEL: cvt_flr_i32_f32_fabs:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cvt_flr_i32_f32_e64 v0, |s6|
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_cvt_flr_i32_f32_e64 v0, |s2|
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: cvt_flr_i32_f32_fabs:
@@ -139,9 +138,7 @@ define amdgpu_kernel void @cvt_flr_i32_f32_fabs(ptr addrspace(1) %out, float %x)
;
; GFX11-SDAG-LABEL: cvt_flr_i32_f32_fabs:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_floor_i32_f32_e64 v1, |s2|
@@ -169,13 +166,14 @@ define amdgpu_kernel void @cvt_flr_i32_f32_fabs(ptr addrspace(1) %out, float %x)
define amdgpu_kernel void @cvt_flr_i32_f32_fneg(ptr addrspace(1) %out, float %x) #0 {
; SI-SDAG-LABEL: cvt_flr_i32_f32_fneg:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cvt_flr_i32_f32_e64 v0, -s6
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_cvt_flr_i32_f32_e64 v0, -s2
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: cvt_flr_i32_f32_fneg:
@@ -192,9 +190,7 @@ define amdgpu_kernel void @cvt_flr_i32_f32_fneg(ptr addrspace(1) %out, float %x)
;
; GFX11-SDAG-LABEL: cvt_flr_i32_f32_fneg:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_floor_i32_f32_e64 v1, -s2
@@ -223,13 +219,14 @@ define amdgpu_kernel void @cvt_flr_i32_f32_fneg(ptr addrspace(1) %out, float %x)
define amdgpu_kernel void @cvt_flr_i32_f32_fabs_fneg(ptr addrspace(1) %out, float %x) #0 {
; SI-SDAG-LABEL: cvt_flr_i32_f32_fabs_fneg:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cvt_flr_i32_f32_e64 v0, -|s6|
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_cvt_flr_i32_f32_e64 v0, -|s2|
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: cvt_flr_i32_f32_fabs_fneg:
@@ -246,9 +243,7 @@ define amdgpu_kernel void @cvt_flr_i32_f32_fabs_fneg(ptr addrspace(1) %out, floa
;
; GFX11-SDAG-LABEL: cvt_flr_i32_f32_fabs_fneg:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_floor_i32_f32_e64 v1, -|s2|
@@ -279,14 +274,15 @@ define amdgpu_kernel void @no_cvt_flr_i32_f32_0(ptr addrspace(1) %out, float %x)
;
; SI-SDAG-LABEL: no_cvt_flr_i32_f32_0:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_floor_f32_e32 v0, s6
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_floor_f32_e32 v0, s2
; SI-SDAG-NEXT: v_cvt_u32_f32_e32 v0, v0
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: no_cvt_flr_i32_f32_0:
@@ -303,9 +299,7 @@ define amdgpu_kernel void @no_cvt_flr_i32_f32_0(ptr addrspace(1) %out, float %x)
;
; GFX11-SDAG-LABEL: no_cvt_flr_i32_f32_0:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_floor_f32_e32 v0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_rpi_i32_f32.ll b/llvm/test/CodeGen/AMDGPU/cvt_rpi_i32_f32.ll
index d359c80ab24c5..147de78f17b9a 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_rpi_i32_f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_rpi_i32_f32.ll
@@ -10,13 +10,14 @@ declare float @llvm.floor.f32(float) #1
define amdgpu_kernel void @cvt_rpi_i32_f32(ptr addrspace(1) %out, float %x) #0 {
; SI-SDAG-LABEL: cvt_rpi_i32_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cvt_rpi_i32_f32_e32 v0, s6
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_cvt_rpi_i32_f32_e32 v0, s2
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: cvt_rpi_i32_f32:
@@ -32,9 +33,7 @@ define amdgpu_kernel void @cvt_rpi_i32_f32(ptr addrspace(1) %out, float %x) #0 {
;
; GFX11-SDAG-LABEL: cvt_rpi_i32_f32:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_nearest_i32_f32_e32 v1, s2
@@ -61,13 +60,14 @@ define amdgpu_kernel void @cvt_rpi_i32_f32(ptr addrspace(1) %out, float %x) #0 {
define amdgpu_kernel void @cvt_rpi_i32_f32_fabs(ptr addrspace(1) %out, float %x) #0 {
; SI-SDAG-LABEL: cvt_rpi_i32_f32_fabs:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cvt_rpi_i32_f32_e64 v0, |s6|
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_cvt_rpi_i32_f32_e64 v0, |s2|
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: cvt_rpi_i32_f32_fabs:
@@ -83,9 +83,7 @@ define amdgpu_kernel void @cvt_rpi_i32_f32_fabs(ptr addrspace(1) %out, float %x)
;
; GFX11-SDAG-LABEL: cvt_rpi_i32_f32_fabs:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_nearest_i32_f32_e64 v1, |s2|
@@ -114,14 +112,15 @@ define amdgpu_kernel void @cvt_rpi_i32_f32_fabs(ptr addrspace(1) %out, float %x)
define amdgpu_kernel void @cvt_rpi_i32_f32_fneg(ptr addrspace(1) %out, float %x) #0 {
; SI-SDAG-LABEL: cvt_rpi_i32_f32_fneg:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_sub_f32_e64 v0, 0.5, s6
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_sub_f32_e64 v0, 0.5, s2
; SI-SDAG-NEXT: v_cvt_flr_i32_f32_e32 v0, v0
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: cvt_rpi_i32_f32_fneg:
@@ -138,9 +137,7 @@ define amdgpu_kernel void @cvt_rpi_i32_f32_fneg(ptr addrspace(1) %out, float %x)
;
; GFX11-SDAG-LABEL: cvt_rpi_i32_f32_fneg:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_sub_f32_e64 v0, 0.5, s2
@@ -174,14 +171,15 @@ define amdgpu_kernel void @cvt_rpi_i32_f32_fabs_fneg(ptr addrspace(1) %out, floa
;
; SI-SDAG-LABEL: cvt_rpi_i32_f32_fabs_fneg:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_sub_f32_e64 v0, 0.5, |s6|
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_sub_f32_e64 v0, 0.5, |s2|
; SI-SDAG-NEXT: v_cvt_flr_i32_f32_e32 v0, v0
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: cvt_rpi_i32_f32_fabs_fneg:
@@ -198,9 +196,7 @@ define amdgpu_kernel void @cvt_rpi_i32_f32_fabs_fneg(ptr addrspace(1) %out, floa
;
; GFX11-SDAG-LABEL: cvt_rpi_i32_f32_fabs_fneg:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_sub_f32_e64 v0, 0.5, |s2|
@@ -233,15 +229,16 @@ define amdgpu_kernel void @cvt_rpi_i32_f32_fabs_fneg(ptr addrspace(1) %out, floa
define amdgpu_kernel void @no_cvt_rpi_i32_f32_0(ptr addrspace(1) %out, float %x) #0 {
; SI-SDAG-LABEL: no_cvt_rpi_i32_f32_0:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_add_f32_e64 v0, s6, 0.5
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_add_f32_e64 v0, s2, 0.5
; SI-SDAG-NEXT: v_floor_f32_e32 v0, v0
; SI-SDAG-NEXT: v_cvt_u32_f32_e32 v0, v0
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: no_cvt_rpi_i32_f32_0:
@@ -259,9 +256,7 @@ define amdgpu_kernel void @no_cvt_rpi_i32_f32_0(ptr addrspace(1) %out, float %x)
;
; GFX11-SDAG-LABEL: no_cvt_rpi_i32_f32_0:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_add_f32_e64 v0, s2, 0.5
diff --git a/llvm/test/CodeGen/AMDGPU/dagcomb-extract-vec-elt-different-sizes.ll b/llvm/test/CodeGen/AMDGPU/dagcomb-extract-vec-elt-different-sizes.ll
index 9f12977a3efde..9b87e491ce278 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcomb-extract-vec-elt-different-sizes.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcomb-extract-vec-elt-different-sizes.ll
@@ -17,23 +17,23 @@ define amdgpu_kernel void @eggs(i1 %arg, ptr addrspace(1) %arg1, ptr %arg2, ptr
; CHECK-NEXT: ; %bb.1: ; %bb10
; CHECK-NEXT: global_load_dwordx2 v[8:9], v0, s[12:13]
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: v_and_b32_e32 v0, 0xff, v8
-; CHECK-NEXT: v_bfe_u32 v1, v8, 8, 8
-; CHECK-NEXT: v_bfe_u32 v2, v8, 16, 8
-; CHECK-NEXT: v_lshrrev_b32_e32 v3, 24, v8
-; CHECK-NEXT: v_and_b32_e32 v4, 0xff, v9
-; CHECK-NEXT: v_bfe_u32 v5, v9, 8, 8
-; CHECK-NEXT: v_bfe_u32 v7, v9, 16, 8
-; CHECK-NEXT: v_lshrrev_b32_e32 v6, 24, v9
+; CHECK-NEXT: v_and_b32_e32 v4, 0xff, v8
+; CHECK-NEXT: v_bfe_u32 v5, v8, 8, 8
+; CHECK-NEXT: v_bfe_u32 v6, v8, 16, 8
+; CHECK-NEXT: v_lshrrev_b32_e32 v7, 24, v8
+; CHECK-NEXT: v_and_b32_e32 v0, 0xff, v9
+; CHECK-NEXT: v_bfe_u32 v1, v9, 8, 8
+; CHECK-NEXT: v_bfe_u32 v3, v9, 16, 8
+; CHECK-NEXT: v_lshrrev_b32_e32 v2, 24, v9
; CHECK-NEXT: s_branch .LBB0_3
; CHECK-NEXT: .LBB0_2:
; CHECK-NEXT: v_mov_b32_e32 v1, 0
-; CHECK-NEXT: v_mov_b32_e32 v2, 0
; CHECK-NEXT: v_mov_b32_e32 v3, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
; CHECK-NEXT: v_mov_b32_e32 v4, 0
; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v7, 0
; CHECK-NEXT: .LBB0_3: ; %bb41
; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x48
; CHECK-NEXT: v_mov_b32_e32 v8, s14
@@ -50,16 +50,16 @@ define amdgpu_kernel void @eggs(i1 %arg, ptr addrspace(1) %arg1, ptr %arg2, ptr
; CHECK-NEXT: v_mov_b32_e32 v19, s25
; CHECK-NEXT: v_mov_b32_e32 v20, s26
; CHECK-NEXT: v_mov_b32_e32 v21, s27
-; CHECK-NEXT: flat_store_byte v[8:9], v0
-; CHECK-NEXT: flat_store_byte v[10:11], v1
-; CHECK-NEXT: flat_store_byte v[12:13], v2
-; CHECK-NEXT: flat_store_byte v[14:15], v3
-; CHECK-NEXT: flat_store_byte v[16:17], v4
-; CHECK-NEXT: flat_store_byte v[18:19], v5
-; CHECK-NEXT: flat_store_byte v[20:21], v7
+; CHECK-NEXT: flat_store_byte v[8:9], v4
+; CHECK-NEXT: flat_store_byte v[10:11], v5
+; CHECK-NEXT: flat_store_byte v[12:13], v6
+; CHECK-NEXT: flat_store_byte v[14:15], v7
+; CHECK-NEXT: flat_store_byte v[16:17], v0
+; CHECK-NEXT: flat_store_byte v[18:19], v1
+; CHECK-NEXT: flat_store_byte v[20:21], v3
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_pk_mov_b32 v[0:1], s[0:1], s[0:1] op_sel:[0,1]
-; CHECK-NEXT: flat_store_byte v[0:1], v6
+; CHECK-NEXT: flat_store_byte v[0:1], v2
; CHECK-NEXT: s_endpgm
bb:
br i1 %arg, label %bb10, label %bb41
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
index 29a8ec57e3951..a9be5ac4040ab 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-select.ll
@@ -59,40 +59,36 @@ define amdgpu_kernel void @select_and3(ptr addrspace(1) %p, i32 %x, i32 %y) {
define amdgpu_kernel void @select_and_v4(ptr addrspace(1) %p, i32 %x, <4 x i32> %y) {
; GFX9-LABEL: select_and_v4:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s8, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_cmp_gt_i32 s8, 10
-; GFX9-NEXT: s_cselect_b32 s3, s3, 0
-; GFX9-NEXT: s_cselect_b32 s2, s2, 0
-; GFX9-NEXT: s_cselect_b32 s1, s1, 0
-; GFX9-NEXT: s_cselect_b32 s0, s0, 0
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX9-NEXT: s_cmp_gt_i32 s10, 10
+; GFX9-NEXT: s_cselect_b32 s0, s15, 0
+; GFX9-NEXT: s_cselect_b32 s1, s14, 0
+; GFX9-NEXT: s_cselect_b32 s2, s13, 0
+; GFX9-NEXT: s_cselect_b32 s3, s12, 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9]
; GFX9-NEXT: s_endpgm
;
; GFX942-LABEL: select_and_v4:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_cmp_gt_i32 s8, 10
-; GFX942-NEXT: s_cselect_b32 s3, s3, 0
-; GFX942-NEXT: s_cselect_b32 s2, s2, 0
-; GFX942-NEXT: s_cselect_b32 s1, s1, 0
-; GFX942-NEXT: s_cselect_b32 s0, s0, 0
-; GFX942-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-NEXT: v_mov_b32_e32 v3, s1
-; GFX942-NEXT: v_mov_b32_e32 v4, s2
-; GFX942-NEXT: v_mov_b32_e32 v5, s3
-; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[6:7]
+; GFX942-NEXT: s_cmp_gt_i32 s10, 10
+; GFX942-NEXT: s_cselect_b32 s0, s15, 0
+; GFX942-NEXT: s_cselect_b32 s1, s14, 0
+; GFX942-NEXT: s_cselect_b32 s2, s13, 0
+; GFX942-NEXT: s_cselect_b32 s3, s12, 0
+; GFX942-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_mov_b32_e32 v4, s1
+; GFX942-NEXT: v_mov_b32_e32 v5, s0
+; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[8:9]
; GFX942-NEXT: s_endpgm
%c = icmp slt i32 %x, 11
%s = select i1 %c, <4 x i32> zeroinitializer, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
@@ -158,40 +154,36 @@ define amdgpu_kernel void @select_or3(ptr addrspace(1) %p, i32 %x, i32 %y) {
define amdgpu_kernel void @select_or_v4(ptr addrspace(1) %p, i32 %x, <4 x i32> %y) {
; GFX9-LABEL: select_or_v4:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s8, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_cmp_lt_i32 s8, 11
-; GFX9-NEXT: s_cselect_b32 s3, s3, -1
-; GFX9-NEXT: s_cselect_b32 s2, s2, -1
-; GFX9-NEXT: s_cselect_b32 s1, s1, -1
-; GFX9-NEXT: s_cselect_b32 s0, s0, -1
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v3, s3
-; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[6:7]
+; GFX9-NEXT: s_cmp_lt_i32 s10, 11
+; GFX9-NEXT: s_cselect_b32 s0, s15, -1
+; GFX9-NEXT: s_cselect_b32 s1, s14, -1
+; GFX9-NEXT: s_cselect_b32 s2, s13, -1
+; GFX9-NEXT: s_cselect_b32 s3, s12, -1
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: global_store_dwordx4 v4, v[0:3], s[8:9]
; GFX9-NEXT: s_endpgm
;
; GFX942-LABEL: select_or_v4:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_cmp_lt_i32 s8, 11
-; GFX942-NEXT: s_cselect_b32 s3, s3, -1
-; GFX942-NEXT: s_cselect_b32 s2, s2, -1
-; GFX942-NEXT: s_cselect_b32 s1, s1, -1
-; GFX942-NEXT: s_cselect_b32 s0, s0, -1
-; GFX942-NEXT: v_mov_b32_e32 v2, s0
-; GFX942-NEXT: v_mov_b32_e32 v3, s1
-; GFX942-NEXT: v_mov_b32_e32 v4, s2
-; GFX942-NEXT: v_mov_b32_e32 v5, s3
-; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[6:7]
+; GFX942-NEXT: s_cmp_lt_i32 s10, 11
+; GFX942-NEXT: s_cselect_b32 s0, s15, -1
+; GFX942-NEXT: s_cselect_b32 s1, s14, -1
+; GFX942-NEXT: s_cselect_b32 s2, s13, -1
+; GFX942-NEXT: s_cselect_b32 s3, s12, -1
+; GFX942-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_mov_b32_e32 v4, s1
+; GFX942-NEXT: v_mov_b32_e32 v5, s0
+; GFX942-NEXT: global_store_dwordx4 v0, v[2:5], s[8:9]
; GFX942-NEXT: s_endpgm
%c = icmp slt i32 %x, 11
%s = select i1 %c, <4 x i32> zeroinitializer, <4 x i32> <i32 -1, i32 -1, i32 -1, i32 -1>
@@ -203,8 +195,7 @@ define amdgpu_kernel void @select_or_v4(ptr addrspace(1) %p, i32 %x, <4 x i32> %
define amdgpu_kernel void @sel_constants_sub_constant_sel_constants(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: sel_constants_sub_constant_sel_constants:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -221,8 +212,7 @@ define amdgpu_kernel void @sel_constants_sub_constant_sel_constants(ptr addrspac
define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_i16(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: sel_constants_sub_constant_sel_constants_i16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -239,8 +229,7 @@ define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_i16(ptr addr
define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_i16_neg(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: sel_constants_sub_constant_sel_constants_i16_neg:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -258,8 +247,7 @@ define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_i16_neg(ptr
define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_v2i16(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: sel_constants_sub_constant_sel_constants_v2i16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -277,8 +265,7 @@ define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_v2i16(ptr ad
define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_v4i32(ptr addrspace(1) %p, i1 %cond) {
; GFX9-LABEL: sel_constants_sub_constant_sel_constants_v4i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bitcmp1_b32 s2, 0
@@ -295,8 +282,7 @@ define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_v4i32(ptr ad
;
; GFX942-LABEL: sel_constants_sub_constant_sel_constants_v4i32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_bitcmp1_b32 s2, 0
@@ -319,8 +305,7 @@ define amdgpu_kernel void @sel_constants_sub_constant_sel_constants_v4i32(ptr ad
define amdgpu_kernel void @sdiv_constant_sel_constants_i64(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: sdiv_constant_sel_constants_i64:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -337,8 +322,7 @@ define amdgpu_kernel void @sdiv_constant_sel_constants_i64(ptr addrspace(1) %p,
define amdgpu_kernel void @sdiv_constant_sel_constants_i32(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: sdiv_constant_sel_constants_i32:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -355,8 +339,7 @@ define amdgpu_kernel void @sdiv_constant_sel_constants_i32(ptr addrspace(1) %p,
define amdgpu_kernel void @udiv_constant_sel_constants_i64(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: udiv_constant_sel_constants_i64:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -373,8 +356,7 @@ define amdgpu_kernel void @udiv_constant_sel_constants_i64(ptr addrspace(1) %p,
define amdgpu_kernel void @srem_constant_sel_constants(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: srem_constant_sel_constants:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -391,8 +373,7 @@ define amdgpu_kernel void @srem_constant_sel_constants(ptr addrspace(1) %p, i1 %
define amdgpu_kernel void @urem_constant_sel_constants(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: urem_constant_sel_constants:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -409,8 +390,7 @@ define amdgpu_kernel void @urem_constant_sel_constants(ptr addrspace(1) %p, i1 %
define amdgpu_kernel void @shl_constant_sel_constants(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: shl_constant_sel_constants:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -427,8 +407,7 @@ define amdgpu_kernel void @shl_constant_sel_constants(ptr addrspace(1) %p, i1 %c
define amdgpu_kernel void @lshr_constant_sel_constants(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: lshr_constant_sel_constants:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -445,8 +424,7 @@ define amdgpu_kernel void @lshr_constant_sel_constants(ptr addrspace(1) %p, i1 %
define amdgpu_kernel void @ashr_constant_sel_constants(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: ashr_constant_sel_constants:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -463,8 +441,7 @@ define amdgpu_kernel void @ashr_constant_sel_constants(ptr addrspace(1) %p, i1 %
define amdgpu_kernel void @fsub_constant_sel_constants(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: fsub_constant_sel_constants:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -482,8 +459,7 @@ define amdgpu_kernel void @fsub_constant_sel_constants(ptr addrspace(1) %p, i1 %
define amdgpu_kernel void @fsub_constant_sel_constants_f16(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: fsub_constant_sel_constants_f16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v1, 0xc400
; GCN-NEXT: v_mov_b32_e32 v2, 0x3c00
; GCN-NEXT: v_mov_b32_e32 v0, 0
@@ -502,8 +478,7 @@ define amdgpu_kernel void @fsub_constant_sel_constants_f16(ptr addrspace(1) %p,
define amdgpu_kernel void @fsub_constant_sel_constants_v2f16(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: fsub_constant_sel_constants_v2f16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -520,11 +495,10 @@ define amdgpu_kernel void @fsub_constant_sel_constants_v2f16(ptr addrspace(1) %p
define amdgpu_kernel void @fsub_constant_sel_constants_v4f32(ptr addrspace(1) %p, i1 %cond) {
; GFX9-LABEL: fsub_constant_sel_constants_v4f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_mov_b32 s3, 0x41500000
; GFX9-NEXT: v_mov_b32_e32 v4, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bitcmp1_b32 s2, 0
; GFX9-NEXT: s_cselect_b32 s2, s3, 0x40c00000
; GFX9-NEXT: s_cselect_b32 s3, 0x41100000, 4.0
@@ -539,11 +513,10 @@ define amdgpu_kernel void @fsub_constant_sel_constants_v4f32(ptr addrspace(1) %p
;
; GFX942-LABEL: fsub_constant_sel_constants_v4f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_mov_b32 s3, 0x41500000
; GFX942-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_bitcmp1_b32 s2, 0
; GFX942-NEXT: s_cselect_b32 s2, s3, 0x40c00000
; GFX942-NEXT: s_cselect_b32 s3, 0x41100000, 4.0
@@ -564,8 +537,7 @@ define amdgpu_kernel void @fsub_constant_sel_constants_v4f32(ptr addrspace(1) %p
define amdgpu_kernel void @fdiv_constant_sel_constants(ptr addrspace(1) %p, i1 %cond) {
; GCN-LABEL: fdiv_constant_sel_constants:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp1_b32 s2, 0
@@ -584,20 +556,20 @@ define amdgpu_kernel void @fdiv_constant_sel_constants(ptr addrspace(1) %p, i1 %
define amdgpu_kernel void @frem_constant_sel_constants(ptr addrspace(1) %p, i1 %cond) {
; GFX9-LABEL: frem_constant_sel_constants:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40400000
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bitcmp1_b32 s0, 0
-; GFX9-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v1, v0, -4.0, s[0:1]
-; GFX9-NEXT: s_mov_b32 s0, 0x40a00000
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, s0
-; GFX9-NEXT: s_and_b64 vcc, exec, s[2:3]
+; GFX9-NEXT: s_bitcmp1_b32 s2, 0
+; GFX9-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX9-NEXT: v_cndmask_b32_e64 v1, v0, -4.0, s[2:3]
+; GFX9-NEXT: s_mov_b32 s2, 0x40a00000
+; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, s2
+; GFX9-NEXT: s_and_b64 vcc, exec, s[4:5]
; GFX9-NEXT: s_cbranch_vccz .LBB26_2
; GFX9-NEXT: ; %bb.1: ; %frem.else
; GFX9-NEXT: v_mov_b32_e32 v0, 0x40a00000
-; GFX9-NEXT: v_cmp_eq_f32_e64 s[0:1], |v1|, s0
-; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[0:1]
+; GFX9-NEXT: v_cmp_eq_f32_e64 s[2:3], |v1|, s2
+; GFX9-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[2:3]
; GFX9-NEXT: s_cbranch_execz .LBB26_3
; GFX9-NEXT: s_branch .LBB26_7
; GFX9-NEXT: .LBB26_2:
@@ -605,7 +577,7 @@ define amdgpu_kernel void @frem_constant_sel_constants(ptr addrspace(1) %p, i1 %
; GFX9-NEXT: .LBB26_3: ; %frem.compute
; GFX9-NEXT: v_frexp_mant_f32_e64 v0, |v1|
; GFX9-NEXT: v_ldexp_f32 v0, v0, 1
-; GFX9-NEXT: v_div_scale_f32 v2, s[0:1], v0, v0, 1.0
+; GFX9-NEXT: v_div_scale_f32 v2, s[2:3], v0, v0, 1.0
; GFX9-NEXT: v_div_scale_f32 v3, vcc, 1.0, v0, 1.0
; GFX9-NEXT: v_rcp_f32_e32 v4, v2
; GFX9-NEXT: v_fma_f32 v5, -v2, v4, 1.0
@@ -637,7 +609,7 @@ define amdgpu_kernel void @frem_constant_sel_constants(ptr addrspace(1) %p, i1 %
; GFX9-NEXT: v_cmp_lt_i32_e32 vcc, 12, v1
; GFX9-NEXT: v_ldexp_f32 v5, v5, 12
; GFX9-NEXT: s_cbranch_vccnz .LBB26_5
-; GFX9-NEXT: .LBB26_6: ; %Flow12
+; GFX9-NEXT: .LBB26_6: ; %Flow13
; GFX9-NEXT: v_add_u32_e32 v1, -11, v1
; GFX9-NEXT: v_ldexp_f32 v1, v4, v1
; GFX9-NEXT: v_mul_f32_e32 v3, v1, v3
@@ -649,30 +621,28 @@ define amdgpu_kernel void @frem_constant_sel_constants(ptr addrspace(1) %p, i1 %
; GFX9-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
; GFX9-NEXT: v_ldexp_f32 v0, v0, v2
; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
-; GFX9-NEXT: .LBB26_7: ; %Flow14
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: .LBB26_7: ; %Flow15
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX942-LABEL: frem_constant_sel_constants:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: v_mov_b32_e32 v0, 0x40400000
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_bitcmp1_b32 s0, 0
-; GFX942-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX942-NEXT: v_cndmask_b32_e64 v1, v0, -4.0, s[0:1]
-; GFX942-NEXT: s_mov_b32 s0, 0x40a00000
-; GFX942-NEXT: v_cmp_ge_f32_e64 s[2:3], |v1|, s0
-; GFX942-NEXT: s_and_b64 vcc, exec, s[2:3]
+; GFX942-NEXT: s_bitcmp1_b32 s2, 0
+; GFX942-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX942-NEXT: v_cndmask_b32_e64 v1, v0, -4.0, s[2:3]
+; GFX942-NEXT: s_mov_b32 s2, 0x40a00000
+; GFX942-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, s2
+; GFX942-NEXT: s_and_b64 vcc, exec, s[4:5]
; GFX942-NEXT: s_cbranch_vccz .LBB26_2
; GFX942-NEXT: ; %bb.1: ; %frem.else
; GFX942-NEXT: v_mov_b32_e32 v0, 0x40a00000
-; GFX942-NEXT: v_cmp_eq_f32_e64 s[0:1], |v1|, s0
+; GFX942-NEXT: v_cmp_eq_f32_e64 s[2:3], |v1|, s2
; GFX942-NEXT: s_nop 1
-; GFX942-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[0:1]
+; GFX942-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[2:3]
; GFX942-NEXT: s_cbranch_execz .LBB26_3
; GFX942-NEXT: s_branch .LBB26_7
; GFX942-NEXT: .LBB26_2:
@@ -680,7 +650,7 @@ define amdgpu_kernel void @frem_constant_sel_constants(ptr addrspace(1) %p, i1 %
; GFX942-NEXT: .LBB26_3: ; %frem.compute
; GFX942-NEXT: v_frexp_mant_f32_e64 v0, |v1|
; GFX942-NEXT: v_ldexp_f32 v0, v0, 1
-; GFX942-NEXT: v_div_scale_f32 v3, s[0:1], v0, v0, 1.0
+; GFX942-NEXT: v_div_scale_f32 v3, s[2:3], v0, v0, 1.0
; GFX942-NEXT: v_rcp_f32_e32 v4, v3
; GFX942-NEXT: v_div_scale_f32 v5, vcc, 1.0, v0, 1.0
; GFX942-NEXT: v_frexp_exp_i32_f32_e32 v2, v1
@@ -713,7 +683,7 @@ define amdgpu_kernel void @frem_constant_sel_constants(ptr addrspace(1) %p, i1 %
; GFX942-NEXT: v_cmp_lt_i32_e32 vcc, 12, v1
; GFX942-NEXT: v_ldexp_f32 v5, v5, 12
; GFX942-NEXT: s_cbranch_vccnz .LBB26_5
-; GFX942-NEXT: .LBB26_6: ; %Flow12
+; GFX942-NEXT: .LBB26_6: ; %Flow13
; GFX942-NEXT: v_add_u32_e32 v1, -11, v1
; GFX942-NEXT: v_ldexp_f32 v1, v4, v1
; GFX942-NEXT: v_mul_f32_e32 v3, v1, v3
@@ -726,10 +696,8 @@ define amdgpu_kernel void @frem_constant_sel_constants(ptr addrspace(1) %p, i1 %
; GFX942-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
; GFX942-NEXT: v_ldexp_f32 v0, v0, v2
; GFX942-NEXT: v_and_b32_e32 v0, 0x7fffffff, v0
-; GFX942-NEXT: .LBB26_7: ; %Flow14
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-NEXT: .LBB26_7: ; %Flow15
; GFX942-NEXT: v_mov_b32_e32 v1, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: global_store_dword v1, v0, s[0:1]
; GFX942-NEXT: s_endpgm
%sel = select i1 %cond, float -4.0, float 3.0
diff --git a/llvm/test/CodeGen/AMDGPU/dbg-info-inline-at.ll b/llvm/test/CodeGen/AMDGPU/dbg-info-inline-at.ll
index 20077fa5d96a7..53bcaa37ef078 100644
--- a/llvm/test/CodeGen/AMDGPU/dbg-info-inline-at.ll
+++ b/llvm/test/CodeGen/AMDGPU/dbg-info-inline-at.ll
@@ -12,20 +12,20 @@ define amdgpu_kernel void @_Z3fooPiiii(ptr addrspace(1) nocapture noundef writeo
; CHECK-NEXT: .cfi_undefined 16
; CHECK-NEXT: .file 1 "." "a.h"
; CHECK-NEXT: .loc 1 5 12 prologue_end ; ./a.h:5:12 @[ a.hip:12:8 ]
-; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CHECK-NEXT: s_load_dword s4, s[8:9], 0x10
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_add_i32 s1, s1, s0
+; CHECK-NEXT: s_add_i32 s3, s3, s2
; CHECK-NEXT: .Ltmp0:
; CHECK-NEXT: .loc 1 5 12 is_stmt 0 ; ./a.h:5:12 @[ a.hip:13:9 ]
-; CHECK-NEXT: s_add_i32 s0, s2, s0
+; CHECK-NEXT: s_add_i32 s2, s4, s2
; CHECK-NEXT: .Ltmp1:
; CHECK-NEXT: .file 2 "a.hip"
; CHECK-NEXT: .loc 2 13 6 is_stmt 1 ; a.hip:13:6
-; CHECK-NEXT: s_mul_i32 s0, s0, s1
-; CHECK-NEXT: v_mov_b32_e32 v1, s0
-; CHECK-NEXT: global_store_dword v0, v1, s[4:5]
+; CHECK-NEXT: s_mul_i32 s2, s2, s3
+; CHECK-NEXT: v_mov_b32_e32 v1, s2
+; CHECK-NEXT: global_store_dword v0, v1, s[0:1]
; CHECK-NEXT: .loc 2 14 1 ; a.hip:14:1
; CHECK-NEXT: s_endpgm
; CHECK-NEXT: .Ltmp2:
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
index a9823e6ee2b08..a37a681dd2512 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-buildvector.ll
@@ -8,10 +8,9 @@
define amdgpu_kernel void @uniform_vec_0_i16(ptr addrspace(1) %out, i16 %a) {
; GCN-LABEL: uniform_vec_0_i16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_lshl_b32 s4, s2, 16
; GCN-NEXT: s_mov_b32 s2, -1
; GCN-NEXT: v_mov_b32_e32 v0, s4
@@ -20,8 +19,7 @@ define amdgpu_kernel void @uniform_vec_0_i16(ptr addrspace(1) %out, i16 %a) {
;
; GFX9-LABEL: uniform_vec_0_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b32 s2, s2, 16
@@ -31,8 +29,7 @@ define amdgpu_kernel void @uniform_vec_0_i16(ptr addrspace(1) %out, i16 %a) {
;
; GFX906-LABEL: uniform_vec_0_i16:
; GFX906: ; %bb.0:
-; GFX906-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX906-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX906-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX906-NEXT: v_mov_b32_e32 v0, 0
; GFX906-NEXT: s_waitcnt lgkmcnt(0)
; GFX906-NEXT: s_lshl_b32 s2, s2, 16
@@ -42,9 +39,7 @@ define amdgpu_kernel void @uniform_vec_0_i16(ptr addrspace(1) %out, i16 %a) {
;
; GFX11-LABEL: uniform_vec_0_i16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_lshl_b32 s2, s2, 16
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -91,10 +86,9 @@ define i32 @divergent_vec_0_i16(i16 %a) {
define amdgpu_kernel void @uniform_vec_i16_0(ptr addrspace(1) %out, i16 %a) {
; GCN-LABEL: uniform_vec_i16_0:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_and_b32 s4, s2, 0xffff
; GCN-NEXT: s_mov_b32 s2, -1
; GCN-NEXT: v_mov_b32_e32 v0, s4
@@ -103,8 +97,7 @@ define amdgpu_kernel void @uniform_vec_i16_0(ptr addrspace(1) %out, i16 %a) {
;
; GFX9-LABEL: uniform_vec_i16_0:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, 0xffff, s2
@@ -114,8 +107,7 @@ define amdgpu_kernel void @uniform_vec_i16_0(ptr addrspace(1) %out, i16 %a) {
;
; GFX906-LABEL: uniform_vec_i16_0:
; GFX906: ; %bb.0:
-; GFX906-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX906-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX906-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX906-NEXT: v_mov_b32_e32 v0, 0
; GFX906-NEXT: s_waitcnt lgkmcnt(0)
; GFX906-NEXT: s_and_b32 s2, 0xffff, s2
@@ -125,9 +117,7 @@ define amdgpu_kernel void @uniform_vec_i16_0(ptr addrspace(1) %out, i16 %a) {
;
; GFX11-LABEL: uniform_vec_i16_0:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, 0xffff, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -180,10 +170,9 @@ define i32 @divergent_vec_i16_0(i16 %a) {
define amdgpu_kernel void @uniform_vec_f16_neg_0(ptr addrspace(1) %out, half %a) {
; GCN-LABEL: uniform_vec_f16_neg_0:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_and_b32 s2, s2, 0xffff
; GCN-NEXT: s_or_b32 s4, s2, 0x80000000
; GCN-NEXT: s_mov_b32 s2, -1
@@ -193,8 +182,7 @@ define amdgpu_kernel void @uniform_vec_f16_neg_0(ptr addrspace(1) %out, half %a)
;
; GFX9-LABEL: uniform_vec_f16_neg_0:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_pack_ll_b32_b16 s2, s2, 0x8000
@@ -204,8 +192,7 @@ define amdgpu_kernel void @uniform_vec_f16_neg_0(ptr addrspace(1) %out, half %a)
;
; GFX906-LABEL: uniform_vec_f16_neg_0:
; GFX906: ; %bb.0:
-; GFX906-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX906-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX906-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX906-NEXT: v_mov_b32_e32 v0, 0
; GFX906-NEXT: s_waitcnt lgkmcnt(0)
; GFX906-NEXT: s_pack_ll_b32_b16 s2, s2, 0x8000
@@ -215,9 +202,7 @@ define amdgpu_kernel void @uniform_vec_f16_neg_0(ptr addrspace(1) %out, half %a)
;
; GFX11-LABEL: uniform_vec_f16_neg_0:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_pack_ll_b32_b16 s2, s2, 0x8000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -280,10 +265,9 @@ define float @divergent_vec_f16_neg_0(half %a) {
define amdgpu_kernel void @uniform_vec_f16_0(ptr addrspace(1) %out, half %a) {
; GCN-LABEL: uniform_vec_f16_0:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_and_b32 s4, s2, 0xffff
; GCN-NEXT: s_mov_b32 s2, -1
; GCN-NEXT: v_mov_b32_e32 v0, s4
@@ -292,8 +276,7 @@ define amdgpu_kernel void @uniform_vec_f16_0(ptr addrspace(1) %out, half %a) {
;
; GFX9-LABEL: uniform_vec_f16_0:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, 0xffff, s2
@@ -303,8 +286,7 @@ define amdgpu_kernel void @uniform_vec_f16_0(ptr addrspace(1) %out, half %a) {
;
; GFX906-LABEL: uniform_vec_f16_0:
; GFX906: ; %bb.0:
-; GFX906-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX906-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX906-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX906-NEXT: v_mov_b32_e32 v0, 0
; GFX906-NEXT: s_waitcnt lgkmcnt(0)
; GFX906-NEXT: s_and_b32 s2, 0xffff, s2
@@ -314,9 +296,7 @@ define amdgpu_kernel void @uniform_vec_f16_0(ptr addrspace(1) %out, half %a) {
;
; GFX11-LABEL: uniform_vec_f16_0:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, 0xffff, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -475,15 +455,16 @@ define amdgpu_kernel void @uniform_vec_i16_LH(ptr addrspace(1) %out, i16 %a, i32
; GCN-LABEL: uniform_vec_i16_LH:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_and_b32 s2, s5, 0xffff0000
-; GCN-NEXT: s_and_b32 s4, s4, 0xffff
-; GCN-NEXT: s_or_b32 s4, s4, s2
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_and_b32 s3, s3, 0xffff0000
+; GCN-NEXT: s_and_b32 s2, s2, 0xffff
+; GCN-NEXT: s_or_b32 s2, s2, s3
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: uniform_vec_i16_LH:
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-sext-inreg.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-sext-inreg.ll
index d8f9bc1a0e054..4c3fd40d7a25a 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-sext-inreg.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-sext-inreg.ll
@@ -5,14 +5,15 @@ define amdgpu_kernel void @uniform_sext_in_reg_i8_to_i32(ptr addrspace(1) %out,
; GCN-LABEL: uniform_sext_in_reg_i8_to_i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_add_i32 s2, s4, s5
-; GCN-NEXT: s_sext_i32_i8 s4, s2
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_add_i32 s2, s2, s3
+; GCN-NEXT: s_sext_i32_i8 s2, s2
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%c = add i32 %a, %b ; add to prevent folding into extload
%shl = shl i32 %c, 24
@@ -25,14 +26,15 @@ define amdgpu_kernel void @divergent_sext_in_reg_i8_to_i32(ptr addrspace(1) %out
; GCN-LABEL: divergent_sext_in_reg_i8_to_i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: s_add_i32 s4, s4, s5
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s4, v0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_add_i32 s0, s2, s3
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; GCN-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%c = add i32 %a, %b ; add to prevent folding into extload
@@ -47,14 +49,15 @@ define amdgpu_kernel void @uniform_sext_in_reg_i16_to_i32(ptr addrspace(1) %out,
; GCN-LABEL: uniform_sext_in_reg_i16_to_i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_add_i32 s2, s4, s5
-; GCN-NEXT: s_sext_i32_i16 s4, s2
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_add_i32 s2, s2, s3
+; GCN-NEXT: s_sext_i32_i16 s2, s2
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%c = add i32 %a, %b ; add to prevent folding into extload
%shl = shl i32 %c, 16
@@ -67,14 +70,15 @@ define amdgpu_kernel void @divergent_sext_in_reg_i16_to_i32(ptr addrspace(1) %ou
; GCN-LABEL: divergent_sext_in_reg_i16_to_i32:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: s_add_i32 s4, s4, s5
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s4, v0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_add_i32 s0, s2, s3
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; GCN-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%c = add i32 %a, %b ; add to prevent folding into extload
diff --git a/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll b/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
index 1bbe503bfe072..7b486b60cb9f3 100644
--- a/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/divergence-driven-trunc-to-i1.ll
@@ -7,26 +7,29 @@ define amdgpu_kernel void @uniform_trunc_i16_to_i1(ptr addrspace(1) %out, i16 %x
; GCN-NEXT: liveins: $sgpr4_sgpr5
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
- ; GCN-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 9, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset, align 4, addrspace 4)
- ; GCN-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 11, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset.align.down, addrspace 4)
- ; GCN-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
- ; GCN-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub0
+ ; GCN-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 9, 0 :: (dereferenceable invariant load (s128) from %ir.out.kernarg.offset, align 4, addrspace 4)
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub1
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
+ ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1
+ ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
- ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, killed [[COPY1]], %subreg.sub1, killed [[S_MOV_B32_1]], %subreg.sub2, killed [[S_MOV_B32_]], %subreg.sub3
+ ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1, killed [[S_MOV_B32_1]], %subreg.sub2, killed [[S_MOV_B32_]], %subreg.sub3
; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 16
- ; GCN-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[S_LOAD_DWORD_IMM]], [[S_MOV_B32_2]], implicit-def dead $scc
- ; GCN-NEXT: [[S_LSHR_B32_:%[0-9]+]]:sreg_32 = S_LSHR_B32 [[S_LOAD_DWORD_IMM]], [[S_MOV_B32_2]], implicit-def dead $scc
- ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY killed [[S_LSHR_B32_]]
- ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY3]], implicit-def dead $scc
+ ; GCN-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY1]], [[S_MOV_B32_2]], implicit-def dead $scc
+ ; GCN-NEXT: [[S_LSHR_B32_:%[0-9]+]]:sreg_32 = S_LSHR_B32 [[COPY1]], [[S_MOV_B32_2]], implicit-def dead $scc
+ ; GCN-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY killed [[S_LSHR_B32_]]
+ ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY6]], implicit-def dead $scc
; GCN-NEXT: S_CMP_EQ_U32 killed [[S_AND_B32_]], 1, implicit-def $scc
- ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_64 = COPY $scc
+ ; GCN-NEXT: [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
; GCN-NEXT: [[S_MOV_B32_3:%[0-9]+]]:sreg_32 = S_MOV_B32 0
; GCN-NEXT: S_CMP_LT_I32 killed [[S_LSHL_B32_]], killed [[S_MOV_B32_3]], implicit-def $scc
- ; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_64 = COPY $scc
- ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY5]], killed [[COPY4]], implicit-def dead $scc
+ ; GCN-NEXT: [[COPY8:%[0-9]+]]:sreg_64 = COPY $scc
+ ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY8]], killed [[COPY7]], implicit-def dead $scc
; GCN-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[S_OR_B64_]], implicit $exec
- ; GCN-NEXT: BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
+ ; GCN-NEXT: BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.2, addrspace 1)
; GCN-NEXT: S_ENDPGM 0
%setcc = icmp slt i16 %x, 0
%select = select i1 %setcc, i1 true, i1 %z
@@ -62,25 +65,27 @@ define amdgpu_kernel void @uniform_trunc_i32_to_i1(ptr addrspace(1) %out, i32 %x
; GCN-NEXT: liveins: $sgpr4_sgpr5
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
- ; GCN-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 9, 0 :: (dereferenceable invariant load (s64) from %ir.out.kernarg.offset, align 4, addrspace 4)
- ; GCN-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[COPY]](p4), 11, 0 :: (dereferenceable invariant load (s64) from %ir.x.kernarg.offset, align 4, addrspace 4)
- ; GCN-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub1
- ; GCN-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX2_IMM]].sub0
+ ; GCN-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 9, 0 :: (dereferenceable invariant load (s128) from %ir.out.kernarg.offset, align 4, addrspace 4)
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX4_IMM]]
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub1
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub0
+ ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1
+ ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
- ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, killed [[COPY1]], %subreg.sub1, killed [[S_MOV_B32_1]], %subreg.sub2, killed [[S_MOV_B32_]], %subreg.sub3
- ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_64 = COPY killed [[S_LOAD_DWORDX2_IMM1]]
- ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[COPY3]].sub0
- ; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY3]].sub1
- ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY5]], implicit-def dead $scc
+ ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1, killed [[S_MOV_B32_1]], %subreg.sub2, killed [[S_MOV_B32_]], %subreg.sub3
+ ; GCN-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub2
+ ; GCN-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub3
+ ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY7]], implicit-def dead $scc
; GCN-NEXT: S_CMP_EQ_U32 killed [[S_AND_B32_]], 1, implicit-def $scc
- ; GCN-NEXT: [[COPY6:%[0-9]+]]:sreg_64 = COPY $scc
+ ; GCN-NEXT: [[COPY8:%[0-9]+]]:sreg_64 = COPY $scc
; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GCN-NEXT: S_CMP_LT_I32 killed [[COPY4]], killed [[S_MOV_B32_2]], implicit-def $scc
- ; GCN-NEXT: [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
- ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY7]], killed [[COPY6]], implicit-def dead $scc
+ ; GCN-NEXT: S_CMP_LT_I32 killed [[COPY6]], killed [[S_MOV_B32_2]], implicit-def $scc
+ ; GCN-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY $scc
+ ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY9]], killed [[COPY8]], implicit-def dead $scc
; GCN-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[S_OR_B64_]], implicit $exec
- ; GCN-NEXT: BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.out.load, addrspace 1)
+ ; GCN-NEXT: BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.2, addrspace 1)
; GCN-NEXT: S_ENDPGM 0
%setcc = icmp slt i32 %x, 0
%select = select i1 %setcc, i1 true, i1 %z
@@ -115,24 +120,25 @@ define amdgpu_kernel void @uniform_trunc_i64_to_i1(ptr addrspace(1) %out, i64 %x
; GCN-NEXT: {{ $}}
; GCN-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; GCN-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 9, 0 :: (dereferenceable invariant load (s128) from %ir.out.kernarg.offset, align 4, addrspace 4)
- ; GCN-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 13, 0 :: (dereferenceable invariant load (s32) from %ir.z.kernarg.offset.align.down, addrspace 4)
- ; GCN-NEXT: [[COPY1:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub1
- ; GCN-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
- ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY2]], %subreg.sub0, killed [[COPY1]], %subreg.sub1
- ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
- ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
+ ; GCN-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 13, 0 :: (dereferenceable invariant load (s32) from %ir.out.kernarg.offset + 16, addrspace 4)
+ ; GCN-NEXT: [[COPY1:%[0-9]+]]:sgpr_128 = COPY killed [[S_LOAD_DWORDX4_IMM]]
+ ; GCN-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub3
+ ; GCN-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY killed [[S_LOAD_DWORD_IMM]]
+ ; GCN-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub1
+ ; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[COPY1]].sub0
+ ; GCN-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1
+ ; GCN-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub1
+ ; GCN-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[REG_SEQUENCE]].sub0
; GCN-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 61440
; GCN-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 -1
- ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY4]], %subreg.sub0, killed [[COPY3]], %subreg.sub1, killed [[S_MOV_B32_1]], %subreg.sub2, killed [[S_MOV_B32_]], %subreg.sub3
- ; GCN-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub3
- ; GCN-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY killed [[S_LOAD_DWORD_IMM]]
- ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY6]], implicit-def dead $scc
+ ; GCN-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sgpr_128 = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1, killed [[S_MOV_B32_1]], %subreg.sub2, killed [[S_MOV_B32_]], %subreg.sub3
+ ; GCN-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 1, killed [[COPY3]], implicit-def dead $scc
; GCN-NEXT: S_CMP_EQ_U32 killed [[S_AND_B32_]], 1, implicit-def $scc
- ; GCN-NEXT: [[COPY7:%[0-9]+]]:sreg_64 = COPY $scc
- ; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
- ; GCN-NEXT: S_CMP_LT_I32 killed [[COPY5]], killed [[S_MOV_B32_2]], implicit-def $scc
; GCN-NEXT: [[COPY8:%[0-9]+]]:sreg_64 = COPY $scc
- ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY8]], killed [[COPY7]], implicit-def dead $scc
+ ; GCN-NEXT: [[S_MOV_B32_2:%[0-9]+]]:sreg_32 = S_MOV_B32 0
+ ; GCN-NEXT: S_CMP_LT_I32 killed [[COPY2]], killed [[S_MOV_B32_2]], implicit-def $scc
+ ; GCN-NEXT: [[COPY9:%[0-9]+]]:sreg_64 = COPY $scc
+ ; GCN-NEXT: [[S_OR_B64_:%[0-9]+]]:sreg_64_xexec = S_OR_B64 killed [[COPY9]], killed [[COPY8]], implicit-def dead $scc
; GCN-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, killed [[S_OR_B64_]], implicit $exec
; GCN-NEXT: BUFFER_STORE_BYTE_OFFSET killed [[V_CNDMASK_B32_e64_]], killed [[REG_SEQUENCE1]], 0, 0, 0, 0, implicit $exec :: (store (s8) into %ir.2, addrspace 1)
; GCN-NEXT: S_ENDPGM 0
diff --git a/llvm/test/CodeGen/AMDGPU/ds-combine-with-dependence.ll b/llvm/test/CodeGen/AMDGPU/ds-combine-with-dependence.ll
index 0497542e11a8b..20c600f6545da 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-combine-with-dependence.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-combine-with-dependence.ll
@@ -36,6 +36,7 @@ define amdgpu_kernel void @ds_combine_nodep(ptr addrspace(1) %out, ptr addrspace
; GCN-LABEL: {{^}}ds_combine_WAR
; GCN: ds_read2_b32 v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}} offset0:7 offset1:27
+; GCN-NEXT: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
; GCN-NEXT: ds_write2_b32 v{{[0-9]+}}, v{{[0-9]+}}, v{{[0-9]+}} offset0:26 offset1:27
define amdgpu_kernel void @ds_combine_WAR(ptr addrspace(1) %out, ptr addrspace(3) %inptr) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
index a895424bc2803..30b41ff702f23 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
@@ -779,16 +779,17 @@ define amdgpu_kernel void @ds_read_diff_base_interleaving(
; GFX1250-LABEL: ds_read_diff_base_interleaving:
; GFX1250: ; %bb.0: ; %bb
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 6, v0 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_and_b32_e32 v1, 0x3ff0, v1
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_add_nc_u32 v2, s0, v1 :: v_dual_add_nc_u32 v3, s1, v0
-; GFX1250-NEXT: v_dual_add_nc_u32 v4, s2, v1 :: v_dual_add_nc_u32 v6, s3, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: v_dual_add_nc_u32 v2, s2, v1 :: v_dual_add_nc_u32 v3, s3, v0
+; GFX1250-NEXT: v_dual_add_nc_u32 v4, s6, v1 :: v_dual_add_nc_u32 v6, s7, v0
; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:1
; GFX1250-NEXT: ds_load_2addr_b32 v[2:3], v3 offset1:4
; GFX1250-NEXT: ds_load_2addr_b32 v[4:5], v4 offset1:1
@@ -802,7 +803,6 @@ define amdgpu_kernel void @ds_read_diff_base_interleaving(
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_dual_sub_f32 v0, v0, v1 :: v_dual_mul_f32 v1, v5, v7
; GFX1250-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_store_b32 v2, v0, s[0:1] offset:40
; GFX1250-NEXT: s_endpgm
ptr addrspace(1) nocapture %arg,
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2.ll b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
index 9f1b55ea3b1ef..ac4106c2033f1 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
@@ -522,12 +522,11 @@ define amdgpu_kernel void @simple_read2_f32_volatile_1(ptr addrspace(1) %out) #0
define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; CI-LABEL: unaligned_read2_f32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_add_i32_e32 v1, vcc, s2, v0
; CI-NEXT: ds_read_u8 v2, v1 offset:1
; CI-NEXT: ds_read_u8 v3, v1 offset:34
@@ -561,8 +560,7 @@ define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrsp
;
; GFX9-ALIGNED-LABEL: unaligned_read2_f32:
; GFX9-ALIGNED: ; %bb.0:
-; GFX9-ALIGNED-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-ALIGNED-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-ALIGNED-NEXT: v_add_u32_e32 v1, s2, v0
@@ -590,12 +588,11 @@ define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrsp
;
; GFX9-UNALIGNED-LABEL: unaligned_read2_f32:
; GFX9-UNALIGNED: ; %bb.0:
-; GFX9-UNALIGNED-NEXT: s_load_dword s0, s[4:5], 0x8
+; GFX9-UNALIGNED-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-UNALIGNED-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-UNALIGNED-NEXT: v_add_u32_e32 v0, s0, v2
+; GFX9-UNALIGNED-NEXT: v_add_u32_e32 v0, s2, v2
; GFX9-UNALIGNED-NEXT: ds_read2_b32 v[0:1], v0 offset1:8
-; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-UNALIGNED-NEXT: v_add_f32_e32 v0, v0, v1
; GFX9-UNALIGNED-NEXT: global_store_dword v2, v0, s[0:1]
@@ -615,12 +612,11 @@ define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrsp
define amdgpu_kernel void @unaligned_offset_read2_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; CI-LABEL: unaligned_offset_read2_f32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_add_i32_e32 v1, vcc, s2, v0
; CI-NEXT: ds_read_u8 v2, v1 offset:6
; CI-NEXT: ds_read_u8 v3, v1 offset:11
@@ -654,8 +650,7 @@ define amdgpu_kernel void @unaligned_offset_read2_f32(ptr addrspace(1) %out, ptr
;
; GFX9-ALIGNED-LABEL: unaligned_offset_read2_f32:
; GFX9-ALIGNED: ; %bb.0:
-; GFX9-ALIGNED-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-ALIGNED-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-ALIGNED-NEXT: v_add_u32_e32 v1, s2, v0
@@ -683,12 +678,11 @@ define amdgpu_kernel void @unaligned_offset_read2_f32(ptr addrspace(1) %out, ptr
;
; GFX9-UNALIGNED-LABEL: unaligned_offset_read2_f32:
; GFX9-UNALIGNED: ; %bb.0:
-; GFX9-UNALIGNED-NEXT: s_load_dword s0, s[4:5], 0x8
+; GFX9-UNALIGNED-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-UNALIGNED-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-UNALIGNED-NEXT: v_add_u32_e32 v0, s0, v2
+; GFX9-UNALIGNED-NEXT: v_add_u32_e32 v0, s2, v2
; GFX9-UNALIGNED-NEXT: ds_read_b64 v[0:1], v0 offset:5
-; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-UNALIGNED-NEXT: v_add_f32_e32 v0, v0, v1
; GFX9-UNALIGNED-NEXT: global_store_dword v2, v0, s[0:1]
@@ -708,41 +702,42 @@ define amdgpu_kernel void @unaligned_offset_read2_f32(ptr addrspace(1) %out, ptr
define amdgpu_kernel void @misaligned_2_simple_read2_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; CI-LABEL: misaligned_2_simple_read2_f32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[4:5], 0x2
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_add_i32_e32 v1, vcc, s0, v0
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: v_add_i32_e32 v1, vcc, s2, v0
; CI-NEXT: ds_read_u16 v2, v1 offset:2
-; CI-NEXT: ds_read_u16 v3, v1 offset:32
-; CI-NEXT: ds_read_u16 v4, v1
-; CI-NEXT: ds_read_u16 v1, v1 offset:34
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: ds_read_u16 v3, v1 offset:34
+; CI-NEXT: ds_read_u16 v4, v1 offset:32
+; CI-NEXT: ds_read_u16 v1, v1
+; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_waitcnt lgkmcnt(3)
; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; CI-NEXT: v_or_b32_e32 v2, v2, v4
-; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_or_b32_e32 v1, v1, v3
-; CI-NEXT: v_add_f32_e32 v2, v2, v1
+; CI-NEXT: s_waitcnt lgkmcnt(2)
+; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_or_b32_e32 v1, v2, v1
+; CI-NEXT: v_or_b32_e32 v2, v3, v4
+; CI-NEXT: v_add_f32_e32 v2, v1, v2
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; CI-NEXT: s_endpgm
;
; GFX9-ALIGNED-LABEL: misaligned_2_simple_read2_f32:
; GFX9-ALIGNED: ; %bb.0:
-; GFX9-ALIGNED-NEXT: s_load_dword s0, s[4:5], 0x8
+; GFX9-ALIGNED-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-ALIGNED-NEXT: v_add_u32_e32 v1, s0, v0
+; GFX9-ALIGNED-NEXT: v_add_u32_e32 v1, s2, v0
; GFX9-ALIGNED-NEXT: ds_read_u16 v2, v1
; GFX9-ALIGNED-NEXT: ds_read_u16 v3, v1 offset:2
; GFX9-ALIGNED-NEXT: ds_read_u16 v4, v1 offset:32
; GFX9-ALIGNED-NEXT: ds_read_u16 v1, v1 offset:34
-; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(2)
; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v2, v3, 16, v2
+; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-ALIGNED-NEXT: v_lshl_or_b32 v1, v1, 16, v4
; GFX9-ALIGNED-NEXT: v_add_f32_e32 v1, v2, v1
; GFX9-ALIGNED-NEXT: global_store_dword v0, v1, s[0:1]
@@ -750,12 +745,11 @@ define amdgpu_kernel void @misaligned_2_simple_read2_f32(ptr addrspace(1) %out,
;
; GFX9-UNALIGNED-LABEL: misaligned_2_simple_read2_f32:
; GFX9-UNALIGNED: ; %bb.0:
-; GFX9-UNALIGNED-NEXT: s_load_dword s0, s[4:5], 0x8
+; GFX9-UNALIGNED-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-UNALIGNED-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-UNALIGNED-NEXT: v_add_u32_e32 v0, s0, v2
+; GFX9-UNALIGNED-NEXT: v_add_u32_e32 v0, s2, v2
; GFX9-UNALIGNED-NEXT: ds_read2_b32 v[0:1], v0 offset1:8
-; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-UNALIGNED-NEXT: v_add_f32_e32 v0, v0, v1
; GFX9-UNALIGNED-NEXT: global_store_dword v2, v0, s[0:1]
@@ -886,16 +880,15 @@ define amdgpu_kernel void @simple_read2_f64_too_far(ptr addrspace(1) %out) #0 {
define amdgpu_kernel void @misaligned_read2_f64(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; CI-LABEL: misaligned_read2_f64:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[4:5], 0x2
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_add_i32_e32 v3, vcc, s0, v0
+; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: v_add_i32_e32 v3, vcc, s2, v0
; CI-NEXT: ds_read2_b32 v[1:2], v3 offset1:1
; CI-NEXT: ds_read2_b32 v[3:4], v3 offset0:14 offset1:15
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT: s_mov_b32 s2, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_add_f64 v[2:3], v[1:2], v[3:4]
; CI-NEXT: v_mov_b32_e32 v1, 0
@@ -904,13 +897,12 @@ define amdgpu_kernel void @misaligned_read2_f64(ptr addrspace(1) %out, ptr addrs
;
; GFX9-LABEL: misaligned_read2_f64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x8
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 3, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v2, s0, v4
+; GFX9-NEXT: v_add_u32_e32 v2, s2, v4
; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset1:1
; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset0:14 offset1:15
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_add_f64 v[0:1], v[0:1], v[2:3]
; GFX9-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
@@ -1169,11 +1161,10 @@ define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(ptr addrspace(1) %C,
define amdgpu_kernel void @misaligned_read2_v2i32(ptr addrspace(1) %out, ptr addrspace(3) %in) #0 {
; CI-LABEL: misaligned_read2_v2i32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
; CI-NEXT: s_mov_b32 s2, -1
@@ -1183,8 +1174,7 @@ define amdgpu_kernel void @misaligned_read2_v2i32(ptr addrspace(1) %out, ptr add
;
; GFX9-LABEL: misaligned_read2_v2i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -1200,11 +1190,10 @@ define amdgpu_kernel void @misaligned_read2_v2i32(ptr addrspace(1) %out, ptr add
define amdgpu_kernel void @misaligned_read2_i64(ptr addrspace(1) %out, ptr addrspace(3) %in) #0 {
; CI-LABEL: misaligned_read2_i64:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_mov_b32_e32 v0, s2
; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
; CI-NEXT: s_mov_b32 s2, -1
@@ -1214,8 +1203,7 @@ define amdgpu_kernel void @misaligned_read2_i64(ptr addrspace(1) %out, ptr addrs
;
; GFX9-LABEL: misaligned_read2_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s2
@@ -1231,16 +1219,16 @@ define amdgpu_kernel void @misaligned_read2_i64(ptr addrspace(1) %out, ptr addrs
define amdgpu_kernel void @ds_read_diff_base_interleaving(
; CI-LABEL: ds_read_diff_base_interleaving:
; CI: ; %bb.0: ; %bb
-; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2
-; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x4
; CI-NEXT: v_lshlrev_b32_e32 v1, 4, v1
; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_add_i32_e32 v2, vcc, s0, v1
-; CI-NEXT: v_add_i32_e32 v3, vcc, s1, v0
-; CI-NEXT: v_add_i32_e32 v4, vcc, s2, v1
-; CI-NEXT: v_add_i32_e32 v6, vcc, s3, v0
+; CI-NEXT: v_add_i32_e32 v2, vcc, s2, v1
+; CI-NEXT: v_add_i32_e32 v3, vcc, s3, v0
+; CI-NEXT: v_add_i32_e32 v4, vcc, s4, v1
+; CI-NEXT: v_add_i32_e32 v6, vcc, s5, v0
; CI-NEXT: ds_read2_b32 v[0:1], v2 offset1:1
; CI-NEXT: ds_read2_b32 v[2:3], v3 offset1:4
; CI-NEXT: ds_read2_b32 v[4:5], v4 offset1:1
@@ -1256,29 +1244,32 @@ define amdgpu_kernel void @ds_read_diff_base_interleaving(
; CI-NEXT: v_mul_f32_e32 v1, v1, v3
; CI-NEXT: v_sub_f32_e32 v0, v0, v1
; CI-NEXT: v_mul_f32_e32 v1, v5, v7
+; CI-NEXT: s_mov_b32 s4, s0
+; CI-NEXT: s_mov_b32 s5, s1
; CI-NEXT: v_sub_f32_e32 v0, v0, v1
; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0 offset:40
; CI-NEXT: s_endpgm
;
; GFX9-LABEL: ds_read_diff_base_interleaving:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x8
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 4, v1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_add_u32_e32 v2, s0, v1
-; GFX9-NEXT: v_add_u32_e32 v3, s1, v0
-; GFX9-NEXT: v_add_u32_e32 v4, s2, v1
-; GFX9-NEXT: v_add_u32_e32 v6, s3, v0
+; GFX9-NEXT: v_add_u32_e32 v2, s2, v1
+; GFX9-NEXT: v_add_u32_e32 v3, s3, v0
+; GFX9-NEXT: v_add_u32_e32 v4, s6, v1
+; GFX9-NEXT: v_add_u32_e32 v6, s7, v0
; GFX9-NEXT: ds_read2_b32 v[0:1], v2 offset1:1
; GFX9-NEXT: ds_read2_b32 v[2:3], v3 offset1:4
; GFX9-NEXT: ds_read2_b32 v[4:5], v4 offset1:1
; GFX9-NEXT: ds_read2_b32 v[6:7], v6 offset1:4
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_waitcnt lgkmcnt(2)
; GFX9-NEXT: v_mul_f32_e32 v0, v0, v2
; GFX9-NEXT: v_add_f32_e32 v0, 2.0, v0
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mul_f32_e32 v2, v4, v6
; GFX9-NEXT: v_sub_f32_e32 v0, v0, v2
; GFX9-NEXT: v_mul_f32_e32 v1, v1, v3
@@ -1330,28 +1321,28 @@ define amdgpu_kernel void @ds_read_call_read(ptr addrspace(1) %out, ptr addrspac
; CI-NEXT: s_getpc_b64 s[48:49]
; CI-NEXT: s_mov_b32 s48, s0
; CI-NEXT: s_load_dwordx4 s[48:51], s[48:49], 0x0
-; CI-NEXT: s_mov_b32 s14, s10
+; CI-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x0
; CI-NEXT: v_lshlrev_b32_e32 v3, 2, v0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_mov_b32 s12, s8
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_add_u32 s48, s48, s11
-; CI-NEXT: s_mov_b64 s[10:11], s[6:7]
-; CI-NEXT: s_load_dwordx2 s[36:37], s[4:5], 0x0
-; CI-NEXT: s_load_dword s6, s[4:5], 0x2
+; CI-NEXT: v_add_i32_e32 v40, vcc, s18, v3
; CI-NEXT: s_addc_u32 s49, s49, 0
+; CI-NEXT: ds_read_b32 v41, v40
; CI-NEXT: s_add_u32 s8, s4, 12
; CI-NEXT: v_lshlrev_b32_e32 v1, 10, v1
+; CI-NEXT: s_mov_b32 s14, s10
; CI-NEXT: s_mov_b32 s13, s9
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_add_i32_e32 v40, vcc, s6, v3
-; CI-NEXT: ds_read_b32 v41, v40
+; CI-NEXT: s_mov_b64 s[10:11], s[6:7]
; CI-NEXT: s_addc_u32 s9, s5, 0
; CI-NEXT: v_lshlrev_b32_e32 v2, 20, v2
; CI-NEXT: v_or_b32_e32 v0, v0, v1
; CI-NEXT: s_mov_b64 s[4:5], s[0:1]
; CI-NEXT: s_mov_b64 s[6:7], s[2:3]
; CI-NEXT: s_mov_b64 s[0:1], s[48:49]
+; CI-NEXT: s_mov_b32 s36, s16
+; CI-NEXT: s_mov_b32 s37, s17
; CI-NEXT: s_mov_b32 s17, void_func_void at abs32@hi
; CI-NEXT: s_mov_b32 s16, void_func_void at abs32@lo
; CI-NEXT: v_or_b32_e32 v31, v0, v2
@@ -1368,39 +1359,37 @@ define amdgpu_kernel void @ds_read_call_read(ptr addrspace(1) %out, ptr addrspac
;
; GFX9-LABEL: ds_read_call_read:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_getpc_b64 s[36:37]
-; GFX9-NEXT: s_mov_b32 s36, s0
-; GFX9-NEXT: s_load_dwordx4 s[36:39], s[36:37], 0x0
-; GFX9-NEXT: s_mov_b32 s14, s10
+; GFX9-NEXT: s_getpc_b64 s[48:49]
+; GFX9-NEXT: s_mov_b32 s48, s0
+; GFX9-NEXT: s_load_dwordx4 s[48:51], s[48:49], 0x0
; GFX9-NEXT: s_mov_b32 s12, s8
+; GFX9-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x0
+; GFX9-NEXT: s_mov_b32 s14, s10
; GFX9-NEXT: s_mov_b32 s13, s9
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_add_u32 s36, s36, s11
-; GFX9-NEXT: s_mov_b64 s[10:11], s[6:7]
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x0
-; GFX9-NEXT: s_addc_u32 s37, s37, 0
+; GFX9-NEXT: s_add_u32 s48, s48, s11
+; GFX9-NEXT: s_addc_u32 s49, s49, 0
+; GFX9-NEXT: v_lshl_add_u32 v41, v0, 2, s38
+; GFX9-NEXT: ds_read_b32 v42, v41
; GFX9-NEXT: s_add_u32 s8, s4, 12
+; GFX9-NEXT: s_mov_b64 s[10:11], s[6:7]
; GFX9-NEXT: s_addc_u32 s9, s5, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_lshl_add_u32 v41, v0, 2, s6
-; GFX9-NEXT: ds_read_b32 v42, v41
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 20, v2
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 10, v1
; GFX9-NEXT: s_mov_b64 s[4:5], s[0:1]
; GFX9-NEXT: s_mov_b64 s[6:7], s[2:3]
-; GFX9-NEXT: s_mov_b64 s[0:1], s[36:37]
+; GFX9-NEXT: s_mov_b64 s[0:1], s[48:49]
; GFX9-NEXT: s_mov_b32 s17, void_func_void at abs32@hi
; GFX9-NEXT: s_mov_b32 s16, void_func_void at abs32@lo
; GFX9-NEXT: v_or3_b32 v31, v0, v1, v2
-; GFX9-NEXT: s_mov_b64 s[2:3], s[38:39]
+; GFX9-NEXT: s_mov_b64 s[2:3], s[50:51]
; GFX9-NEXT: s_mov_b32 s32, 0
; GFX9-NEXT: v_mov_b32_e32 v40, 0
; GFX9-NEXT: s_swappc_b64 s[30:31], s[16:17]
; GFX9-NEXT: ds_read_b32 v0, v41 offset:4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v0, v42, v0
-; GFX9-NEXT: global_store_dword v40, v0, s[34:35]
+; GFX9-NEXT: global_store_dword v40, v0, s[36:37]
; GFX9-NEXT: s_endpgm
%x = call i32 @llvm.amdgcn.workitem.id.x()
%arrayidx0 = getelementptr i32, ptr addrspace(3) %arg, i32 %x
diff --git a/llvm/test/CodeGen/AMDGPU/ds_write2.ll b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
index b346d5ddb79aa..e78e9c184d670 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_write2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
@@ -797,15 +797,15 @@ define amdgpu_kernel void @simple_write2_one_val_f64(ptr addrspace(1) %C, ptr ad
define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(ptr addrspace(1) %C, ptr addrspace(1) %in, ptr addrspace(3) %lds) #0 {
; CI-LABEL: misaligned_simple_write2_one_val_f64:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
-; CI-NEXT: s_load_dword s4, s[4:5], 0x4
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, 0
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: buffer_load_dwordx2 v[1:2], v[0:1], s[0:3], 0 addr64
-; CI-NEXT: v_add_i32_e32 v0, vcc, s4, v0
+; CI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; CI-NEXT: buffer_load_dwordx2 v[1:2], v[0:1], s[4:7], 0 addr64
+; CI-NEXT: v_add_i32_e32 v0, vcc, s2, v0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: ds_write2_b32 v0, v1, v2 offset1:1
@@ -814,8 +814,7 @@ define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(ptr addrspace(1)
;
; GFX9-LABEL: misaligned_simple_write2_one_val_f64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x10
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x8
; GFX9-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
@@ -854,15 +853,15 @@ define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(ptr addrspace(1)
define amdgpu_kernel void @unaligned_offset_simple_write2_one_val_f64(ptr addrspace(1) %C, ptr addrspace(1) %in, ptr addrspace(3) %lds) #0 {
; CI-LABEL: unaligned_offset_simple_write2_one_val_f64:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2
-; CI-NEXT: s_load_dword s4, s[4:5], 0x4
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, 0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, 0
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: buffer_load_dwordx2 v[1:2], v[0:1], s[0:3], 0 addr64
-; CI-NEXT: v_add_i32_e32 v0, vcc, s4, v0
+; CI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; CI-NEXT: buffer_load_dwordx2 v[1:2], v[0:1], s[4:7], 0 addr64
+; CI-NEXT: v_add_i32_e32 v0, vcc, s2, v0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: ds_write_b8 v0, v1 offset:9
@@ -887,8 +886,7 @@ define amdgpu_kernel void @unaligned_offset_simple_write2_one_val_f64(ptr addrsp
;
; GFX9-ALIGNED-LABEL: unaligned_offset_simple_write2_one_val_f64:
; GFX9-ALIGNED: ; %bb.0:
-; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX9-ALIGNED-NEXT: s_load_dword s2, s[4:5], 0x10
+; GFX9-ALIGNED-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x8
; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-ALIGNED-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
@@ -914,8 +912,7 @@ define amdgpu_kernel void @unaligned_offset_simple_write2_one_val_f64(ptr addrsp
;
; GFX9-UNALIGNED-LABEL: unaligned_offset_simple_write2_one_val_f64:
; GFX9-UNALIGNED: ; %bb.0:
-; GFX9-UNALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX9-UNALIGNED-NEXT: s_load_dword s2, s[4:5], 0x10
+; GFX9-UNALIGNED-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x8
; GFX9-UNALIGNED-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX9-UNALIGNED-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-UNALIGNED-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
index c381b1806acdf..af76ad771c92f 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_dynelt.ll
@@ -5,20 +5,19 @@
define amdgpu_kernel void @float4_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: float4_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s6, 1
-; GCN-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GCN-NEXT: s_cmp_lg_u32 s6, 2
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1.0, s[2:3]
-; GCN-NEXT: s_cselect_b64 vcc, -1, 0
-; GCN-NEXT: s_cmp_lg_u32 s6, 3
-; GCN-NEXT: v_cndmask_b32_e32 v0, 2.0, v0, vcc
-; GCN-NEXT: s_cselect_b64 vcc, -1, 0
-; GCN-NEXT: v_cndmask_b32_e32 v2, 4.0, v0, vcc
+; GCN-NEXT: s_cmp_eq_u32 s2, 1
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GCN-NEXT: s_cmp_lg_u32 s2, 2
+; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s[0:1]
+; GCN-NEXT: s_cselect_b64 vcc, -1, 0
+; GCN-NEXT: s_cmp_lg_u32 s2, 3
+; GCN-NEXT: v_cndmask_b32_e32 v2, 2.0, v2, vcc
+; GCN-NEXT: s_cselect_b64 vcc, -1, 0
+; GCN-NEXT: v_cndmask_b32_e32 v2, 4.0, v2, vcc
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
;
@@ -58,18 +57,17 @@ entry:
define amdgpu_kernel void @int4_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: int4_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_eq_u32 s2, 1
-; GCN-NEXT: s_cselect_b32 s3, 1, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 2
-; GCN-NEXT: s_cselect_b32 s3, s3, 2
+; GCN-NEXT: s_cselect_b32 s0, s0, 2
; GCN-NEXT: s_cmp_lg_u32 s2, 3
-; GCN-NEXT: s_cselect_b32 s2, s3, 4
-; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, s0, 4
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
;
@@ -109,25 +107,24 @@ entry:
define amdgpu_kernel void @double4_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: double4_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_mov_b32 s3, 0x3ff028f5
; GCN-NEXT: s_mov_b32 s4, 0xc28f5c29
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_eq_u32 s2, 1
-; GCN-NEXT: s_cselect_b32 s3, s3, 0x3f847ae1
-; GCN-NEXT: s_cselect_b32 s4, s4, 0x47ae147b
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_cselect_b32 s0, s3, 0x3f847ae1
+; GCN-NEXT: s_cselect_b32 s1, s4, 0x47ae147b
; GCN-NEXT: s_cmp_eq_u32 s2, 2
-; GCN-NEXT: s_cselect_b32 s4, 0xe147ae14, s4
-; GCN-NEXT: s_cselect_b32 s3, 0x4000147a, s3
+; GCN-NEXT: s_cselect_b32 s1, 0xe147ae14, s1
+; GCN-NEXT: s_cselect_b32 s0, 0x4000147a, s0
; GCN-NEXT: s_cmp_eq_u32 s2, 3
-; GCN-NEXT: s_cselect_b32 s2, 0x40100a3d, s3
-; GCN-NEXT: s_cselect_b32 s3, 0x70a3d70a, s4
-; GCN-NEXT: v_mov_b32_e32 v3, s1
-; GCN-NEXT: v_mov_b32_e32 v0, s3
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: s_cselect_b32 s0, 0x40100a3d, s0
+; GCN-NEXT: s_cselect_b32 s1, 0x70a3d70a, s1
+; GCN-NEXT: v_mov_b32_e32 v2, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN-NEXT: s_endpgm
;
; GCN-O0-LABEL: double4_extelt:
@@ -210,33 +207,32 @@ entry:
define amdgpu_kernel void @double5_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: double5_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s2, 0x3ff028f5
-; GCN-NEXT: s_mov_b32 s3, 0xc28f5c29
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s6, 1
-; GCN-NEXT: s_cselect_b32 s2, s2, 0x3f847ae1
-; GCN-NEXT: s_cselect_b32 s3, s3, 0x47ae147b
-; GCN-NEXT: s_cmp_eq_u32 s6, 2
-; GCN-NEXT: s_cselect_b32 s8, 0xe147ae14, s3
-; GCN-NEXT: s_cselect_b32 s7, 0x4000147a, s2
-; GCN-NEXT: s_cmp_eq_u32 s6, 3
+; GCN-NEXT: s_mov_b32 s3, 0x3ff028f5
+; GCN-NEXT: s_mov_b32 s4, 0xc28f5c29
+; GCN-NEXT: s_cmp_eq_u32 s2, 1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_cselect_b32 s0, s3, 0x3f847ae1
+; GCN-NEXT: s_cselect_b32 s1, s4, 0x47ae147b
+; GCN-NEXT: s_cmp_eq_u32 s2, 2
+; GCN-NEXT: s_cselect_b32 s6, 0xe147ae14, s1
+; GCN-NEXT: s_cselect_b32 s3, 0x4000147a, s0
+; GCN-NEXT: s_cmp_eq_u32 s2, 3
+; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GCN-NEXT: s_and_b64 s[4:5], s[0:1], exec
+; GCN-NEXT: s_cselect_b32 s7, 0x40100a3d, s3
+; GCN-NEXT: s_cmp_eq_u32 s2, 4
; GCN-NEXT: s_cselect_b64 s[2:3], -1, 0
; GCN-NEXT: s_and_b64 s[4:5], s[2:3], exec
-; GCN-NEXT: s_cselect_b32 s9, 0x40100a3d, s7
-; GCN-NEXT: s_cmp_eq_u32 s6, 4
-; GCN-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN-NEXT: s_and_b64 s[6:7], s[4:5], exec
-; GCN-NEXT: s_cselect_b32 s6, 0x40140a3d, s9
-; GCN-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
-; GCN-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; GCN-NEXT: s_cselect_b32 s2, 0x70a3d70a, s8
-; GCN-NEXT: v_mov_b32_e32 v3, s1
-; GCN-NEXT: v_mov_b32_e32 v0, s2
-; GCN-NEXT: v_mov_b32_e32 v1, s6
+; GCN-NEXT: s_cselect_b32 s4, 0x40140a3d, s7
+; GCN-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
+; GCN-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; GCN-NEXT: s_cselect_b32 s0, 0x70a3d70a, s6
; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: v_mov_b32_e32 v3, s4
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN-NEXT: s_endpgm
;
; GCN-O0-LABEL: double5_extelt:
@@ -360,16 +356,15 @@ entry:
define amdgpu_kernel void @half4_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: half4_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s2, 0x40003c00
-; GCN-NEXT: s_mov_b32 s3, 0x44004200
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s4, 0x40003c00
+; GCN-NEXT: s_mov_b32 s5, 0x44004200
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 s4, s6, 4
-; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_lshl_b32 s0, s2, 4
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_lshr_b64 s[0:1], s[4:5], s0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_short v[0:1], v2
; GCN-NEXT: s_endpgm
;
@@ -403,14 +398,13 @@ entry:
define amdgpu_kernel void @float2_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: float2_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_eq_u32 s2, 1
-; GCN-NEXT: s_cselect_b64 s[2:3], -1, 0
; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s[2:3]
; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s[0:1]
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
;
@@ -444,19 +438,18 @@ entry:
define amdgpu_kernel void @double2_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: double2_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_mov_b32 s3, 0x3ff028f5
; GCN-NEXT: s_mov_b32 s4, 0xc28f5c29
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_eq_u32 s2, 1
-; GCN-NEXT: s_cselect_b32 s2, s3, 0x3f847ae1
-; GCN-NEXT: s_cselect_b32 s3, s4, 0x47ae147b
-; GCN-NEXT: v_mov_b32_e32 v3, s1
-; GCN-NEXT: v_mov_b32_e32 v0, s3
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_cselect_b32 s0, s3, 0x3f847ae1
+; GCN-NEXT: s_cselect_b32 s1, s4, 0x47ae147b
+; GCN-NEXT: v_mov_b32_e32 v2, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN-NEXT: s_endpgm
;
; GCN-O0-LABEL: double2_extelt:
@@ -515,40 +508,39 @@ entry:
define amdgpu_kernel void @half8_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: half8_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: v_mov_b32_e32 v0, 0x3c00
-; GCN-NEXT: v_mov_b32_e32 v1, 0x4000
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: v_mov_b32_e32 v2, 0x3c00
+; GCN-NEXT: v_mov_b32_e32 v3, 0x4000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_eq_u32 s2, 1
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 2
-; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-NEXT: v_mov_b32_e32 v1, 0x4200
+; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-NEXT: v_mov_b32_e32 v3, 0x4200
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 3
-; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-NEXT: v_mov_b32_e32 v1, 0x4400
+; GCN-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GCN-NEXT: v_mov_b32_e32 v3, 0x4400
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 4
-; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-NEXT: v_mov_b32_e32 v1, 0x4500
+; GCN-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GCN-NEXT: v_mov_b32_e32 v3, 0x4500
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 5
-; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-NEXT: v_mov_b32_e32 v1, 0x4600
+; GCN-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GCN-NEXT: v_mov_b32_e32 v3, 0x4600
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 6
-; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-NEXT: v_mov_b32_e32 v1, 0x4700
+; GCN-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GCN-NEXT: v_mov_b32_e32 v3, 0x4700
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 7
-; GCN-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GCN-NEXT: v_mov_b32_e32 v1, 0x4800
+; GCN-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; GCN-NEXT: v_mov_b32_e32 v3, 0x4800
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
-; GCN-NEXT: v_cndmask_b32_e32 v2, v1, v0, vcc
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
; GCN-NEXT: flat_store_short v[0:1], v2
; GCN-NEXT: s_endpgm
;
@@ -618,26 +610,25 @@ entry:
define amdgpu_kernel void @short8_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: short8_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_eq_u32 s2, 1
-; GCN-NEXT: s_cselect_b32 s3, 2, 1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, 2, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 2
-; GCN-NEXT: s_cselect_b32 s3, s3, 3
+; GCN-NEXT: s_cselect_b32 s0, s0, 3
; GCN-NEXT: s_cmp_lg_u32 s2, 3
-; GCN-NEXT: s_cselect_b32 s3, s3, 4
+; GCN-NEXT: s_cselect_b32 s0, s0, 4
; GCN-NEXT: s_cmp_lg_u32 s2, 4
-; GCN-NEXT: s_cselect_b32 s3, s3, 5
+; GCN-NEXT: s_cselect_b32 s0, s0, 5
; GCN-NEXT: s_cmp_lg_u32 s2, 5
-; GCN-NEXT: s_cselect_b32 s3, s3, 6
+; GCN-NEXT: s_cselect_b32 s0, s0, 6
; GCN-NEXT: s_cmp_lg_u32 s2, 6
-; GCN-NEXT: s_cselect_b32 s3, s3, 7
+; GCN-NEXT: s_cselect_b32 s0, s0, 7
; GCN-NEXT: s_cmp_lg_u32 s2, 7
-; GCN-NEXT: s_cselect_b32 s2, s3, 8
-; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, s0, 8
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_short v[0:1], v2
; GCN-NEXT: s_endpgm
;
@@ -686,22 +677,21 @@ entry:
define amdgpu_kernel void @float8_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: float8_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 1.0
; GCN-NEXT: v_mov_b32_e32 v1, 2.0
; GCN-NEXT: v_mov_b32_e32 v2, 0x40400000
; GCN-NEXT: v_mov_b32_e32 v3, 4.0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_mov_b32 m0, s2
+; GCN-NEXT: v_mov_b32_e32 v8, s0
+; GCN-NEXT: v_mov_b32_e32 v9, s1
; GCN-NEXT: v_mov_b32_e32 v4, 0x40a00000
; GCN-NEXT: v_mov_b32_e32 v5, 0x40c00000
; GCN-NEXT: v_mov_b32_e32 v6, 0x40e00000
; GCN-NEXT: v_mov_b32_e32 v7, 0x41000000
-; GCN-NEXT: v_movrels_b32_e32 v2, v0
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: flat_store_dword v[0:1], v2
+; GCN-NEXT: v_movrels_b32_e32 v0, v0
+; GCN-NEXT: flat_store_dword v[8:9], v0
; GCN-NEXT: s_endpgm
;
; GCN-O0-LABEL: float8_extelt:
@@ -752,8 +742,7 @@ entry:
define amdgpu_kernel void @double8_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: double8_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s18, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
; GCN-NEXT: s_mov_b32 s0, 0
; GCN-NEXT: s_mov_b32 s15, 0x40200000
; GCN-NEXT: s_mov_b32 s13, 0x401c0000
@@ -772,9 +761,10 @@ define amdgpu_kernel void @double8_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-NEXT: s_mov_b32 s14, s0
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v15, s15
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_lshl_b32 m0, s18, 1
+; GCN-NEXT: v_mov_b32_e32 v16, s16
+; GCN-NEXT: v_mov_b32_e32 v17, s17
; GCN-NEXT: v_mov_b32_e32 v2, s2
; GCN-NEXT: v_mov_b32_e32 v3, s3
; GCN-NEXT: v_mov_b32_e32 v4, s4
@@ -788,11 +778,10 @@ define amdgpu_kernel void @double8_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-NEXT: v_mov_b32_e32 v12, s12
; GCN-NEXT: v_mov_b32_e32 v13, s13
; GCN-NEXT: v_mov_b32_e32 v14, s14
-; GCN-NEXT: v_movrels_b32_e32 v16, v1
-; GCN-NEXT: v_movrels_b32_e32 v15, v0
-; GCN-NEXT: v_mov_b32_e32 v0, s16
-; GCN-NEXT: v_mov_b32_e32 v1, s17
-; GCN-NEXT: flat_store_dwordx2 v[0:1], v[15:16]
+; GCN-NEXT: v_mov_b32_e32 v15, s15
+; GCN-NEXT: v_movrels_b32_e32 v19, v1
+; GCN-NEXT: v_movrels_b32_e32 v18, v0
+; GCN-NEXT: flat_store_dwordx2 v[16:17], v[18:19]
; GCN-NEXT: s_endpgm
;
; GCN-O0-LABEL: double8_extelt:
@@ -904,8 +893,7 @@ entry:
define amdgpu_kernel void @double7_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: double7_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx2 s[14:15], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s16, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
; GCN-NEXT: s_mov_b32 s0, 0
; GCN-NEXT: s_mov_b32 s13, 0x401c0000
; GCN-NEXT: s_mov_b32 s11, 0x40180000
@@ -920,11 +908,12 @@ define amdgpu_kernel void @double7_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-NEXT: s_mov_b32 s8, s0
; GCN-NEXT: s_mov_b32 s10, s0
; GCN-NEXT: s_mov_b32 s12, s0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v15, s15
-; GCN-NEXT: s_lshl_b32 m0, s16, 1
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_lshl_b32 m0, s18, 1
+; GCN-NEXT: v_mov_b32_e32 v16, s16
+; GCN-NEXT: v_mov_b32_e32 v17, s17
; GCN-NEXT: v_mov_b32_e32 v2, s2
; GCN-NEXT: v_mov_b32_e32 v3, s3
; GCN-NEXT: v_mov_b32_e32 v4, s4
@@ -938,11 +927,10 @@ define amdgpu_kernel void @double7_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-NEXT: v_mov_b32_e32 v12, s12
; GCN-NEXT: v_mov_b32_e32 v13, s13
; GCN-NEXT: v_mov_b32_e32 v14, s14
-; GCN-NEXT: v_movrels_b32_e32 v16, v1
-; GCN-NEXT: v_movrels_b32_e32 v15, v0
-; GCN-NEXT: v_mov_b32_e32 v0, s14
-; GCN-NEXT: v_mov_b32_e32 v1, s15
-; GCN-NEXT: flat_store_dwordx2 v[0:1], v[15:16]
+; GCN-NEXT: v_mov_b32_e32 v15, s15
+; GCN-NEXT: v_movrels_b32_e32 v19, v1
+; GCN-NEXT: v_movrels_b32_e32 v18, v0
+; GCN-NEXT: flat_store_dwordx2 v[16:17], v[18:19]
; GCN-NEXT: s_endpgm
;
; GCN-O0-LABEL: double7_extelt:
@@ -1054,14 +1042,15 @@ entry:
define amdgpu_kernel void @float16_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: float16_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 1.0
; GCN-NEXT: v_mov_b32_e32 v1, 2.0
; GCN-NEXT: v_mov_b32_e32 v2, 0x40400000
; GCN-NEXT: v_mov_b32_e32 v3, 4.0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_mov_b32 m0, s2
+; GCN-NEXT: v_mov_b32_e32 v16, s0
+; GCN-NEXT: v_mov_b32_e32 v17, s1
; GCN-NEXT: v_mov_b32_e32 v4, 0x40a00000
; GCN-NEXT: v_mov_b32_e32 v5, 0x40c00000
; GCN-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -1074,10 +1063,8 @@ define amdgpu_kernel void @float16_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-NEXT: v_mov_b32_e32 v13, 0x41600000
; GCN-NEXT: v_mov_b32_e32 v14, 0x41700000
; GCN-NEXT: v_mov_b32_e32 v15, 0x41800000
-; GCN-NEXT: v_movrels_b32_e32 v2, v0
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: flat_store_dword v[0:1], v2
+; GCN-NEXT: v_movrels_b32_e32 v0, v0
+; GCN-NEXT: flat_store_dword v[16:17], v0
; GCN-NEXT: s_endpgm
;
; GCN-O0-LABEL: float16_extelt:
@@ -1152,76 +1139,75 @@ entry:
define amdgpu_kernel void @double15_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: double15_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s36, 0
-; GCN-NEXT: s_mov_b32 s65, 0x402e0000
-; GCN-NEXT: s_mov_b32 s63, 0x402c0000
-; GCN-NEXT: s_mov_b32 s61, 0x402a0000
-; GCN-NEXT: s_mov_b32 s59, 0x40280000
-; GCN-NEXT: s_mov_b32 s57, 0x40260000
-; GCN-NEXT: s_mov_b32 s55, 0x40240000
-; GCN-NEXT: s_mov_b32 s53, 0x40220000
-; GCN-NEXT: s_mov_b32 s51, 0x40200000
-; GCN-NEXT: s_mov_b32 s49, 0x401c0000
-; GCN-NEXT: s_mov_b32 s47, 0x40180000
-; GCN-NEXT: s_mov_b32 s45, 0x40140000
-; GCN-NEXT: s_mov_b32 s43, 0x40100000
-; GCN-NEXT: s_mov_b32 s41, 0x40080000
-; GCN-NEXT: s_mov_b32 s39, 2.0
-; GCN-NEXT: s_mov_b32 s37, 0x3ff00000
-; GCN-NEXT: s_mov_b32 s38, s36
-; GCN-NEXT: s_mov_b32 s40, s36
-; GCN-NEXT: s_mov_b32 s42, s36
-; GCN-NEXT: s_mov_b32 s44, s36
-; GCN-NEXT: s_mov_b32 s46, s36
-; GCN-NEXT: s_mov_b32 s48, s36
-; GCN-NEXT: s_mov_b32 s50, s36
-; GCN-NEXT: s_mov_b32 s52, s36
-; GCN-NEXT: s_mov_b32 s54, s36
-; GCN-NEXT: s_mov_b32 s56, s36
-; GCN-NEXT: s_mov_b32 s58, s36
-; GCN-NEXT: s_mov_b32 s60, s36
-; GCN-NEXT: s_mov_b32 s62, s36
-; GCN-NEXT: s_mov_b32 s64, s36
-; GCN-NEXT: v_mov_b32_e32 v0, s36
-; GCN-NEXT: v_mov_b32_e32 v1, s37
-; GCN-NEXT: v_mov_b32_e32 v31, s67
+; GCN-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s0, 0
+; GCN-NEXT: s_mov_b32 s29, 0x402e0000
+; GCN-NEXT: s_mov_b32 s27, 0x402c0000
+; GCN-NEXT: s_mov_b32 s25, 0x402a0000
+; GCN-NEXT: s_mov_b32 s23, 0x40280000
+; GCN-NEXT: s_mov_b32 s21, 0x40260000
+; GCN-NEXT: s_mov_b32 s19, 0x40240000
+; GCN-NEXT: s_mov_b32 s17, 0x40220000
+; GCN-NEXT: s_mov_b32 s15, 0x40200000
+; GCN-NEXT: s_mov_b32 s13, 0x401c0000
+; GCN-NEXT: s_mov_b32 s11, 0x40180000
+; GCN-NEXT: s_mov_b32 s9, 0x40140000
+; GCN-NEXT: s_mov_b32 s7, 0x40100000
+; GCN-NEXT: s_mov_b32 s5, 0x40080000
+; GCN-NEXT: s_mov_b32 s3, 2.0
+; GCN-NEXT: s_mov_b32 s1, 0x3ff00000
+; GCN-NEXT: s_mov_b32 s2, s0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s6, s0
+; GCN-NEXT: s_mov_b32 s8, s0
+; GCN-NEXT: s_mov_b32 s10, s0
+; GCN-NEXT: s_mov_b32 s12, s0
+; GCN-NEXT: s_mov_b32 s14, s0
+; GCN-NEXT: s_mov_b32 s16, s0
+; GCN-NEXT: s_mov_b32 s18, s0
+; GCN-NEXT: s_mov_b32 s20, s0
+; GCN-NEXT: s_mov_b32 s22, s0
+; GCN-NEXT: s_mov_b32 s24, s0
+; GCN-NEXT: s_mov_b32 s26, s0
+; GCN-NEXT: s_mov_b32 s28, s0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v31, s31
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 m0, s2, 1
-; GCN-NEXT: v_mov_b32_e32 v2, s38
-; GCN-NEXT: v_mov_b32_e32 v3, s39
-; GCN-NEXT: v_mov_b32_e32 v4, s40
-; GCN-NEXT: v_mov_b32_e32 v5, s41
-; GCN-NEXT: v_mov_b32_e32 v6, s42
-; GCN-NEXT: v_mov_b32_e32 v7, s43
-; GCN-NEXT: v_mov_b32_e32 v8, s44
-; GCN-NEXT: v_mov_b32_e32 v9, s45
-; GCN-NEXT: v_mov_b32_e32 v10, s46
-; GCN-NEXT: v_mov_b32_e32 v11, s47
-; GCN-NEXT: v_mov_b32_e32 v12, s48
-; GCN-NEXT: v_mov_b32_e32 v13, s49
-; GCN-NEXT: v_mov_b32_e32 v14, s50
-; GCN-NEXT: v_mov_b32_e32 v15, s51
-; GCN-NEXT: v_mov_b32_e32 v16, s52
-; GCN-NEXT: v_mov_b32_e32 v17, s53
-; GCN-NEXT: v_mov_b32_e32 v18, s54
-; GCN-NEXT: v_mov_b32_e32 v19, s55
-; GCN-NEXT: v_mov_b32_e32 v20, s56
-; GCN-NEXT: v_mov_b32_e32 v21, s57
-; GCN-NEXT: v_mov_b32_e32 v22, s58
-; GCN-NEXT: v_mov_b32_e32 v23, s59
-; GCN-NEXT: v_mov_b32_e32 v24, s60
-; GCN-NEXT: v_mov_b32_e32 v25, s61
-; GCN-NEXT: v_mov_b32_e32 v26, s62
-; GCN-NEXT: v_mov_b32_e32 v27, s63
-; GCN-NEXT: v_mov_b32_e32 v28, s64
-; GCN-NEXT: v_mov_b32_e32 v29, s65
-; GCN-NEXT: v_mov_b32_e32 v30, s66
+; GCN-NEXT: s_lshl_b32 m0, s38, 1
+; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: v_mov_b32_e32 v3, s3
+; GCN-NEXT: v_mov_b32_e32 v4, s4
+; GCN-NEXT: v_mov_b32_e32 v5, s5
+; GCN-NEXT: v_mov_b32_e32 v6, s6
+; GCN-NEXT: v_mov_b32_e32 v7, s7
+; GCN-NEXT: v_mov_b32_e32 v8, s8
+; GCN-NEXT: v_mov_b32_e32 v9, s9
+; GCN-NEXT: v_mov_b32_e32 v10, s10
+; GCN-NEXT: v_mov_b32_e32 v11, s11
+; GCN-NEXT: v_mov_b32_e32 v12, s12
+; GCN-NEXT: v_mov_b32_e32 v13, s13
+; GCN-NEXT: v_mov_b32_e32 v14, s14
+; GCN-NEXT: v_mov_b32_e32 v15, s15
+; GCN-NEXT: v_mov_b32_e32 v16, s16
+; GCN-NEXT: v_mov_b32_e32 v17, s17
+; GCN-NEXT: v_mov_b32_e32 v18, s18
+; GCN-NEXT: v_mov_b32_e32 v19, s19
+; GCN-NEXT: v_mov_b32_e32 v20, s20
+; GCN-NEXT: v_mov_b32_e32 v21, s21
+; GCN-NEXT: v_mov_b32_e32 v22, s22
+; GCN-NEXT: v_mov_b32_e32 v23, s23
+; GCN-NEXT: v_mov_b32_e32 v24, s24
+; GCN-NEXT: v_mov_b32_e32 v25, s25
+; GCN-NEXT: v_mov_b32_e32 v26, s26
+; GCN-NEXT: v_mov_b32_e32 v27, s27
+; GCN-NEXT: v_mov_b32_e32 v28, s28
+; GCN-NEXT: v_mov_b32_e32 v29, s29
+; GCN-NEXT: v_mov_b32_e32 v30, s30
; GCN-NEXT: v_movrels_b32_e32 v32, v1
; GCN-NEXT: v_movrels_b32_e32 v31, v0
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s36
+; GCN-NEXT: v_mov_b32_e32 v1, s37
; GCN-NEXT: flat_store_dwordx2 v[0:1], v[31:32]
; GCN-NEXT: s_endpgm
;
@@ -1414,78 +1400,77 @@ entry:
define amdgpu_kernel void @double16_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: double16_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s36, 0
-; GCN-NEXT: s_mov_b32 s67, 0x40300000
-; GCN-NEXT: s_mov_b32 s65, 0x402e0000
-; GCN-NEXT: s_mov_b32 s63, 0x402c0000
-; GCN-NEXT: s_mov_b32 s61, 0x402a0000
-; GCN-NEXT: s_mov_b32 s59, 0x40280000
-; GCN-NEXT: s_mov_b32 s57, 0x40260000
-; GCN-NEXT: s_mov_b32 s55, 0x40240000
-; GCN-NEXT: s_mov_b32 s53, 0x40220000
-; GCN-NEXT: s_mov_b32 s51, 0x40200000
-; GCN-NEXT: s_mov_b32 s49, 0x401c0000
-; GCN-NEXT: s_mov_b32 s47, 0x40180000
-; GCN-NEXT: s_mov_b32 s45, 0x40140000
-; GCN-NEXT: s_mov_b32 s43, 0x40100000
-; GCN-NEXT: s_mov_b32 s41, 0x40080000
-; GCN-NEXT: s_mov_b32 s39, 2.0
-; GCN-NEXT: s_mov_b32 s37, 0x3ff00000
-; GCN-NEXT: s_mov_b32 s38, s36
-; GCN-NEXT: s_mov_b32 s40, s36
-; GCN-NEXT: s_mov_b32 s42, s36
-; GCN-NEXT: s_mov_b32 s44, s36
-; GCN-NEXT: s_mov_b32 s46, s36
-; GCN-NEXT: s_mov_b32 s48, s36
-; GCN-NEXT: s_mov_b32 s50, s36
-; GCN-NEXT: s_mov_b32 s52, s36
-; GCN-NEXT: s_mov_b32 s54, s36
-; GCN-NEXT: s_mov_b32 s56, s36
-; GCN-NEXT: s_mov_b32 s58, s36
-; GCN-NEXT: s_mov_b32 s60, s36
-; GCN-NEXT: s_mov_b32 s62, s36
-; GCN-NEXT: s_mov_b32 s64, s36
-; GCN-NEXT: s_mov_b32 s66, s36
-; GCN-NEXT: v_mov_b32_e32 v0, s36
-; GCN-NEXT: v_mov_b32_e32 v1, s37
-; GCN-NEXT: v_mov_b32_e32 v31, s67
+; GCN-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s0, 0
+; GCN-NEXT: s_mov_b32 s31, 0x40300000
+; GCN-NEXT: s_mov_b32 s29, 0x402e0000
+; GCN-NEXT: s_mov_b32 s27, 0x402c0000
+; GCN-NEXT: s_mov_b32 s25, 0x402a0000
+; GCN-NEXT: s_mov_b32 s23, 0x40280000
+; GCN-NEXT: s_mov_b32 s21, 0x40260000
+; GCN-NEXT: s_mov_b32 s19, 0x40240000
+; GCN-NEXT: s_mov_b32 s17, 0x40220000
+; GCN-NEXT: s_mov_b32 s15, 0x40200000
+; GCN-NEXT: s_mov_b32 s13, 0x401c0000
+; GCN-NEXT: s_mov_b32 s11, 0x40180000
+; GCN-NEXT: s_mov_b32 s9, 0x40140000
+; GCN-NEXT: s_mov_b32 s7, 0x40100000
+; GCN-NEXT: s_mov_b32 s5, 0x40080000
+; GCN-NEXT: s_mov_b32 s3, 2.0
+; GCN-NEXT: s_mov_b32 s1, 0x3ff00000
+; GCN-NEXT: s_mov_b32 s2, s0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s6, s0
+; GCN-NEXT: s_mov_b32 s8, s0
+; GCN-NEXT: s_mov_b32 s10, s0
+; GCN-NEXT: s_mov_b32 s12, s0
+; GCN-NEXT: s_mov_b32 s14, s0
+; GCN-NEXT: s_mov_b32 s16, s0
+; GCN-NEXT: s_mov_b32 s18, s0
+; GCN-NEXT: s_mov_b32 s20, s0
+; GCN-NEXT: s_mov_b32 s22, s0
+; GCN-NEXT: s_mov_b32 s24, s0
+; GCN-NEXT: s_mov_b32 s26, s0
+; GCN-NEXT: s_mov_b32 s28, s0
+; GCN-NEXT: s_mov_b32 s30, s0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v31, s31
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 m0, s2, 1
-; GCN-NEXT: v_mov_b32_e32 v2, s38
-; GCN-NEXT: v_mov_b32_e32 v3, s39
-; GCN-NEXT: v_mov_b32_e32 v4, s40
-; GCN-NEXT: v_mov_b32_e32 v5, s41
-; GCN-NEXT: v_mov_b32_e32 v6, s42
-; GCN-NEXT: v_mov_b32_e32 v7, s43
-; GCN-NEXT: v_mov_b32_e32 v8, s44
-; GCN-NEXT: v_mov_b32_e32 v9, s45
-; GCN-NEXT: v_mov_b32_e32 v10, s46
-; GCN-NEXT: v_mov_b32_e32 v11, s47
-; GCN-NEXT: v_mov_b32_e32 v12, s48
-; GCN-NEXT: v_mov_b32_e32 v13, s49
-; GCN-NEXT: v_mov_b32_e32 v14, s50
-; GCN-NEXT: v_mov_b32_e32 v15, s51
-; GCN-NEXT: v_mov_b32_e32 v16, s52
-; GCN-NEXT: v_mov_b32_e32 v17, s53
-; GCN-NEXT: v_mov_b32_e32 v18, s54
-; GCN-NEXT: v_mov_b32_e32 v19, s55
-; GCN-NEXT: v_mov_b32_e32 v20, s56
-; GCN-NEXT: v_mov_b32_e32 v21, s57
-; GCN-NEXT: v_mov_b32_e32 v22, s58
-; GCN-NEXT: v_mov_b32_e32 v23, s59
-; GCN-NEXT: v_mov_b32_e32 v24, s60
-; GCN-NEXT: v_mov_b32_e32 v25, s61
-; GCN-NEXT: v_mov_b32_e32 v26, s62
-; GCN-NEXT: v_mov_b32_e32 v27, s63
-; GCN-NEXT: v_mov_b32_e32 v28, s64
-; GCN-NEXT: v_mov_b32_e32 v29, s65
-; GCN-NEXT: v_mov_b32_e32 v30, s66
+; GCN-NEXT: s_lshl_b32 m0, s38, 1
+; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: v_mov_b32_e32 v3, s3
+; GCN-NEXT: v_mov_b32_e32 v4, s4
+; GCN-NEXT: v_mov_b32_e32 v5, s5
+; GCN-NEXT: v_mov_b32_e32 v6, s6
+; GCN-NEXT: v_mov_b32_e32 v7, s7
+; GCN-NEXT: v_mov_b32_e32 v8, s8
+; GCN-NEXT: v_mov_b32_e32 v9, s9
+; GCN-NEXT: v_mov_b32_e32 v10, s10
+; GCN-NEXT: v_mov_b32_e32 v11, s11
+; GCN-NEXT: v_mov_b32_e32 v12, s12
+; GCN-NEXT: v_mov_b32_e32 v13, s13
+; GCN-NEXT: v_mov_b32_e32 v14, s14
+; GCN-NEXT: v_mov_b32_e32 v15, s15
+; GCN-NEXT: v_mov_b32_e32 v16, s16
+; GCN-NEXT: v_mov_b32_e32 v17, s17
+; GCN-NEXT: v_mov_b32_e32 v18, s18
+; GCN-NEXT: v_mov_b32_e32 v19, s19
+; GCN-NEXT: v_mov_b32_e32 v20, s20
+; GCN-NEXT: v_mov_b32_e32 v21, s21
+; GCN-NEXT: v_mov_b32_e32 v22, s22
+; GCN-NEXT: v_mov_b32_e32 v23, s23
+; GCN-NEXT: v_mov_b32_e32 v24, s24
+; GCN-NEXT: v_mov_b32_e32 v25, s25
+; GCN-NEXT: v_mov_b32_e32 v26, s26
+; GCN-NEXT: v_mov_b32_e32 v27, s27
+; GCN-NEXT: v_mov_b32_e32 v28, s28
+; GCN-NEXT: v_mov_b32_e32 v29, s29
+; GCN-NEXT: v_mov_b32_e32 v30, s30
; GCN-NEXT: v_movrels_b32_e32 v32, v1
; GCN-NEXT: v_movrels_b32_e32 v31, v0
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s36
+; GCN-NEXT: v_mov_b32_e32 v1, s37
; GCN-NEXT: flat_store_dwordx2 v[0:1], v[31:32]
; GCN-NEXT: s_endpgm
;
@@ -1678,14 +1663,13 @@ entry:
define amdgpu_kernel void @float32_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: float32_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 1.0
; GCN-NEXT: v_mov_b32_e32 v1, 2.0
; GCN-NEXT: v_mov_b32_e32 v2, 0x40400000
+; GCN-NEXT: v_mov_b32_e32 v3, 4.0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_mov_b32 m0, s2
-; GCN-NEXT: v_mov_b32_e32 v3, 4.0
; GCN-NEXT: v_mov_b32_e32 v4, 0x40a00000
; GCN-NEXT: v_mov_b32_e32 v5, 0x40c00000
; GCN-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -1840,16 +1824,15 @@ entry:
define amdgpu_kernel void @byte8_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: byte8_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s2, 0x4030201
-; GCN-NEXT: s_mov_b32 s3, 0x8070605
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s4, 0x4030201
+; GCN-NEXT: s_mov_b32 s5, 0x8070605
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 s4, s6, 3
-; GCN-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_lshl_b32 s0, s2, 3
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_lshr_b64 s[0:1], s[4:5], s0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_byte v[0:1], v2
; GCN-NEXT: s_endpgm
;
@@ -1908,42 +1891,41 @@ entry:
define amdgpu_kernel void @byte16_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: byte16_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_eq_u32 s2, 1
-; GCN-NEXT: s_cselect_b32 s3, 2, 1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, 2, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 2
-; GCN-NEXT: s_cselect_b32 s3, s3, 3
+; GCN-NEXT: s_cselect_b32 s0, s0, 3
; GCN-NEXT: s_cmp_lg_u32 s2, 3
-; GCN-NEXT: s_cselect_b32 s3, s3, 4
+; GCN-NEXT: s_cselect_b32 s0, s0, 4
; GCN-NEXT: s_cmp_lg_u32 s2, 4
-; GCN-NEXT: s_cselect_b32 s3, s3, 5
+; GCN-NEXT: s_cselect_b32 s0, s0, 5
; GCN-NEXT: s_cmp_lg_u32 s2, 5
-; GCN-NEXT: s_cselect_b32 s3, s3, 6
+; GCN-NEXT: s_cselect_b32 s0, s0, 6
; GCN-NEXT: s_cmp_lg_u32 s2, 6
-; GCN-NEXT: s_cselect_b32 s3, s3, 7
+; GCN-NEXT: s_cselect_b32 s0, s0, 7
; GCN-NEXT: s_cmp_lg_u32 s2, 7
-; GCN-NEXT: s_cselect_b32 s3, s3, 8
+; GCN-NEXT: s_cselect_b32 s0, s0, 8
; GCN-NEXT: s_cmp_lg_u32 s2, 8
-; GCN-NEXT: s_cselect_b32 s3, s3, 9
+; GCN-NEXT: s_cselect_b32 s0, s0, 9
; GCN-NEXT: s_cmp_lg_u32 s2, 9
-; GCN-NEXT: s_cselect_b32 s3, s3, 10
+; GCN-NEXT: s_cselect_b32 s0, s0, 10
; GCN-NEXT: s_cmp_lg_u32 s2, 10
-; GCN-NEXT: s_cselect_b32 s3, s3, 11
+; GCN-NEXT: s_cselect_b32 s0, s0, 11
; GCN-NEXT: s_cmp_lg_u32 s2, 11
-; GCN-NEXT: s_cselect_b32 s3, s3, 12
+; GCN-NEXT: s_cselect_b32 s0, s0, 12
; GCN-NEXT: s_cmp_lg_u32 s2, 12
-; GCN-NEXT: s_cselect_b32 s3, s3, 13
+; GCN-NEXT: s_cselect_b32 s0, s0, 13
; GCN-NEXT: s_cmp_lg_u32 s2, 13
-; GCN-NEXT: s_cselect_b32 s3, s3, 14
+; GCN-NEXT: s_cselect_b32 s0, s0, 14
; GCN-NEXT: s_cmp_lg_u32 s2, 14
-; GCN-NEXT: s_cselect_b32 s3, s3, 15
+; GCN-NEXT: s_cselect_b32 s0, s0, 15
; GCN-NEXT: s_cmp_lg_u32 s2, 15
-; GCN-NEXT: s_cselect_b32 s2, s3, 16
-; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, s0, 16
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_byte v[0:1], v2
; GCN-NEXT: s_endpgm
;
@@ -2037,15 +2019,14 @@ entry:
define amdgpu_kernel void @bit4_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: bit4_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 s2, s2, 3
-; GCN-NEXT: s_lshr_b32 s2, 0x1000100, s2
-; GCN-NEXT: s_and_b32 s2, s2, 1
; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_lshl_b32 s0, s2, 3
+; GCN-NEXT: s_lshr_b32 s0, 0x1000100, s0
+; GCN-NEXT: s_and_b32 s0, s0, 1
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
;
@@ -2085,269 +2066,268 @@ entry:
define amdgpu_kernel void @bit128_extelt(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: bit128_extelt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_cmp_lg_u32 s2, 1
-; GCN-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GCN-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
; GCN-NEXT: s_cmp_lg_u32 s2, 2
-; GCN-NEXT: v_readfirstlane_b32 s3, v0
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: v_readfirstlane_b32 s0, v2
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 3
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 4
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 5
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 6
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 7
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 8
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 9
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 10
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 11
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 12
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 13
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 14
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 15
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 16
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 17
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 18
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 19
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 20
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 21
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 22
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 23
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 24
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 25
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 26
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 27
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 28
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 29
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 30
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 31
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 32
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 33
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 34
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 35
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 36
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 37
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 38
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 39
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 40
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 41
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 42
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 43
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 44
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 45
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 46
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 47
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 48
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 49
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 50
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 51
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 52
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 53
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 54
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 55
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 56
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 57
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 58
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 59
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 60
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 61
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 62
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmp_lg_u32 s2, 63
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmp_lg_u32 s2, 64
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x41
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x42
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x43
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x44
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x45
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x46
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x47
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x48
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x49
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x4a
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x4b
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x4c
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x4d
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x4e
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x4f
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x50
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x51
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x52
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x53
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x54
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x55
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x56
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x57
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x58
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x59
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x5a
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x5b
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x5c
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x5d
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x5e
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x5f
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x60
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x61
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x62
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x63
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x64
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x65
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x66
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x67
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x68
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x69
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x6a
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x6b
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x6c
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x6d
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x6e
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x6f
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x70
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x71
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x72
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x73
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x74
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x75
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x76
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x77
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x78
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x79
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x7a
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x7b
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x7c
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x7d
-; GCN-NEXT: s_cselect_b32 s3, s3, 0
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x7e
-; GCN-NEXT: s_cselect_b32 s3, s3, 1
+; GCN-NEXT: s_cselect_b32 s0, s0, 1
; GCN-NEXT: s_cmpk_lg_i32 s2, 0x7f
-; GCN-NEXT: s_cselect_b32 s2, s3, 0
-; GCN-NEXT: s_and_b32 s2, s2, 1
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_cselect_b32 s0, s0, 0
+; GCN-NEXT: s_and_b32 s0, s0, 1
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-f16.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-f16.ll
index a36be284a4bc3..1a8b6e047402e 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-f16.ll
@@ -216,34 +216,19 @@ define amdgpu_kernel void @extract_vector_elt_v3f16(ptr addrspace(1) %out, <3 x
; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 offset:2
; VI-NEXT: s_endpgm
;
-; GFX11-TRUE16-LABEL: extract_vector_elt_v3f16:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX11-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX11-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: buffer_store_b16 v1, off, s[4:7], 0
-; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0 offset:2
-; GFX11-TRUE16-NEXT: s_endpgm
-;
-; GFX11-FAKE16-LABEL: extract_vector_elt_v3f16:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX11-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s2
-; GFX11-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX11-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], 0
-; GFX11-FAKE16-NEXT: buffer_store_b16 v1, off, s[4:7], 0 offset:2
-; GFX11-FAKE16-NEXT: s_endpgm
+; GFX11-LABEL: extract_vector_elt_v3f16:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-NEXT: s_mov_b32 s6, -1
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: s_mov_b32 s4, s0
+; GFX11-NEXT: s_mov_b32 s5, s1
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: buffer_store_b16 v0, off, s[4:7], 0
+; GFX11-NEXT: buffer_store_b16 v1, off, s[4:7], 0 offset:2
+; GFX11-NEXT: s_endpgm
%p0 = extractelement <3 x half> %foo, i32 0
%p1 = extractelement <3 x half> %foo, i32 2
%out1 = getelementptr half, ptr addrspace(1) %out, i32 1
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll
index 0d8a9f6aca34b..e135b1134932e 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i16.ll
@@ -186,31 +186,16 @@ define amdgpu_kernel void @dynamic_extract_vector_elt_v3i16(ptr addrspace(1) %ou
;
; GCN-LABEL: dynamic_extract_vector_elt_v3i16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x15
-; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x13
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x13
+; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_lshl_b32 s2, s2, 4
-; GCN-NEXT: s_lshr_b64 s[4:5], s[6:7], s2
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GCN-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: buffer_store_short v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
-;
-; GFX89-LABEL: dynamic_extract_vector_elt_v3i16:
-; GFX89: ; %bb.0:
-; GFX89-NEXT: s_load_dword s8, s[4:5], 0x54
-; GFX89-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x4c
-; GFX89-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX89-NEXT: s_mov_b32 s3, 0xf000
-; GFX89-NEXT: s_mov_b32 s2, -1
-; GFX89-NEXT: s_waitcnt lgkmcnt(0)
-; GFX89-NEXT: s_lshl_b32 s4, s8, 4
-; GFX89-NEXT: s_lshr_b64 s[4:5], s[6:7], s4
-; GFX89-NEXT: v_mov_b32_e32 v0, s4
-; GFX89-NEXT: buffer_store_short v0, off, s[0:3], 0
-; GFX89-NEXT: s_endpgm
%p0 = extractelement <3 x i16> %foo, i32 %idx
%out1 = getelementptr i16, ptr addrspace(1) %out, i32 1
store i16 %p0, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll
index eb0ed5e6ee4b9..d8bb7dee9c2e6 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i64.ll
@@ -81,8 +81,6 @@ define amdgpu_kernel void @dyn_extract_vector_elt_v3i64(ptr addrspace(1) %out, <
; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
; GCN-DAG: s_cmp_eq_u32 s{{[0-9]+}}, 3
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
-; GCN-DAG: s_cselect_b32 s{{[0-9]+}}, s{{[0-9]+}}, s{{[0-9]+}}
; GCN: store_dwordx2 v[{{[0-9:]+}}]
define amdgpu_kernel void @dyn_extract_vector_elt_v4i64(ptr addrspace(1) %out, <4 x i64> %foo, i32 %elt) #0 {
%dynelt = extractelement <4 x i64> %foo, i32 %elt
diff --git a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
index 033ab8dc0a0ae..107984c236fb8 100644
--- a/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract_vector_elt-i8.ll
@@ -8,8 +8,7 @@ define amdgpu_kernel void @extract_vector_elt_v1i8(ptr addrspace(1) %out, <1 x i
; SI-NEXT: s_mov_b32 flat_scratch_lo, s13
; SI-NEXT: s_add_i32 s12, s12, s17
; SI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_load_dword s2, s[8:9], 0x2
+; SI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
@@ -19,8 +18,7 @@ define amdgpu_kernel void @extract_vector_elt_v1i8(ptr addrspace(1) %out, <1 x i
;
; VI-LABEL: extract_vector_elt_v1i8:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -41,8 +39,7 @@ define amdgpu_kernel void @extract_vector_elt_v2i8(ptr addrspace(1) %out, <2 x i
; SI-NEXT: s_mov_b32 flat_scratch_lo, s13
; SI-NEXT: s_add_i32 s12, s12, s17
; SI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_load_dword s2, s[8:9], 0x2
+; SI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s3, s2, 8
; SI-NEXT: v_mov_b32_e32 v0, s0
@@ -61,8 +58,7 @@ define amdgpu_kernel void @extract_vector_elt_v2i8(ptr addrspace(1) %out, <2 x i
;
; VI-LABEL: extract_vector_elt_v2i8:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
@@ -95,8 +91,7 @@ define amdgpu_kernel void @extract_vector_elt_v3i8(ptr addrspace(1) %out, <3 x i
; SI-NEXT: s_mov_b32 flat_scratch_lo, s13
; SI-NEXT: s_add_i32 s12, s12, s17
; SI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_load_dword s2, s[8:9], 0x2
+; SI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s3, s2, 16
; SI-NEXT: v_mov_b32_e32 v0, s0
@@ -115,8 +110,7 @@ define amdgpu_kernel void @extract_vector_elt_v3i8(ptr addrspace(1) %out, <3 x i
;
; VI-LABEL: extract_vector_elt_v3i8:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
@@ -149,8 +143,7 @@ define amdgpu_kernel void @extract_vector_elt_v4i8(ptr addrspace(1) %out, <4 x i
; SI-NEXT: s_mov_b32 flat_scratch_lo, s13
; SI-NEXT: s_add_i32 s12, s12, s17
; SI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_load_dword s2, s[8:9], 0x2
+; SI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_lshr_b32 s3, s2, 16
; SI-NEXT: v_mov_b32_e32 v0, s0
@@ -169,8 +162,7 @@ define amdgpu_kernel void @extract_vector_elt_v4i8(ptr addrspace(1) %out, <4 x i
;
; VI-LABEL: extract_vector_elt_v4i8:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
index 9018d71e7a59c..a21002951ae21 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
@@ -12,38 +12,35 @@
define amdgpu_kernel void @s_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {
; CI-LABEL: s_fabs_free_bf16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s2, s2, 0x7fff
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0x7fff
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_short v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_fabs_free_bf16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x7fff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fabs_free_bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff
@@ -53,9 +50,7 @@ define amdgpu_kernel void @s_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {
;
; GFX11-TRUE16-LABEL: s_fabs_free_bf16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0x7fff
@@ -66,9 +61,7 @@ define amdgpu_kernel void @s_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {
;
; GFX11-FAKE16-LABEL: s_fabs_free_bf16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 0x7fff
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -84,38 +77,35 @@ define amdgpu_kernel void @s_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {
define amdgpu_kernel void @s_fabs_bf16(ptr addrspace(1) %out, bfloat %in) {
; CI-LABEL: s_fabs_bf16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s2, s2, 0x7fff
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0x7fff
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_short v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_fabs_bf16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x7fff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fabs_bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff
@@ -125,9 +115,7 @@ define amdgpu_kernel void @s_fabs_bf16(ptr addrspace(1) %out, bfloat %in) {
;
; GFX11-TRUE16-LABEL: s_fabs_bf16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0x7fff
@@ -138,9 +126,7 @@ define amdgpu_kernel void @s_fabs_bf16(ptr addrspace(1) %out, bfloat %in) {
;
; GFX11-FAKE16-LABEL: s_fabs_bf16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 0x7fff
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -155,38 +141,35 @@ define amdgpu_kernel void @s_fabs_bf16(ptr addrspace(1) %out, bfloat %in) {
define amdgpu_kernel void @s_fabs_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) {
; CI-LABEL: s_fabs_v2bf16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0x7fff7fff
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_fabs_v2bf16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff7fff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fabs_v2bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff7fff
@@ -196,9 +179,7 @@ define amdgpu_kernel void @s_fabs_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in
;
; GFX11-LABEL: s_fabs_v2bf16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -218,13 +199,13 @@ define amdgpu_kernel void @s_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s3, s3, 0x7fff7fff
-; CI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v0, s2
-; CI-NEXT: v_mov_b32_e32 v1, s3
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: s_and_b32 s0, s3, 0x7fff7fff
+; CI-NEXT: s_and_b32 s1, s2, 0x7fff7fff
+; CI-NEXT: v_mov_b32_e32 v2, s1
+; CI-NEXT: v_mov_b32_e32 v3, s0
+; CI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_fabs_v4bf16:
@@ -234,23 +215,23 @@ define amdgpu_kernel void @s_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s4, s3, 0x7fff
-; VI-NEXT: s_lshr_b32 s3, s3, 16
-; VI-NEXT: s_and_b32 s5, s2, 0x7fff
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_lshr_b32 s1, s3, 16
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s3, 0x7fff
+; VI-NEXT: s_and_b32 s1, s1, 0x7fff
+; VI-NEXT: s_and_b32 s0, 0xffff, s0
+; VI-NEXT: s_lshl_b32 s1, s1, 16
+; VI-NEXT: s_or_b32 s0, s0, s1
+; VI-NEXT: s_and_b32 s1, s2, 0x7fff
; VI-NEXT: s_lshr_b32 s2, s2, 16
-; VI-NEXT: s_and_b32 s3, s3, 0x7fff
; VI-NEXT: s_and_b32 s2, s2, 0x7fff
-; VI-NEXT: s_and_b32 s4, 0xffff, s4
-; VI-NEXT: s_and_b32 s5, 0xffff, s5
-; VI-NEXT: s_lshl_b32 s3, s3, 16
+; VI-NEXT: s_and_b32 s1, 0xffff, s1
; VI-NEXT: s_lshl_b32 s2, s2, 16
-; VI-NEXT: s_or_b32 s3, s4, s3
-; VI-NEXT: s_or_b32 s2, s5, s2
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_or_b32 s1, s1, s2
+; VI-NEXT: v_mov_b32_e32 v2, s1
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fabs_v4bf16:
@@ -296,52 +277,49 @@ define amdgpu_kernel void @s_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat> %in
define amdgpu_kernel void @fabs_fold_bf16(ptr addrspace(1) %out, bfloat %in0, bfloat %in1) {
; CI-LABEL: fabs_fold_bf16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s3, s2, 0x7fff
-; CI-NEXT: s_and_b32 s2, s2, 0xffff0000
-; CI-NEXT: s_lshl_b32 s3, s3, 16
-; CI-NEXT: v_mov_b32_e32 v0, s2
-; CI-NEXT: v_mul_f32_e32 v0, s3, v0
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: s_and_b32 s0, s2, 0x7fff
+; CI-NEXT: s_and_b32 s1, s2, 0xffff0000
+; CI-NEXT: s_lshl_b32 s0, s0, 16
+; CI-NEXT: v_mov_b32_e32 v2, s1
+; CI-NEXT: v_mul_f32_e32 v2, s0, v2
+; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; CI-NEXT: flat_store_short v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: fabs_fold_bf16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s3, s2, 0x7fff
-; VI-NEXT: s_and_b32 s2, s2, 0xffff0000
-; VI-NEXT: s_lshl_b32 s3, s3, 16
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mul_f32_e32 v0, s3, v0
-; VI-NEXT: v_bfe_u32 v1, v0, 16, 1
-; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v0
-; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
-; VI-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff
+; VI-NEXT: s_and_b32 s1, s2, 0xffff0000
+; VI-NEXT: s_lshl_b32 s0, s0, 16
+; VI-NEXT: v_mov_b32_e32 v2, s1
+; VI-NEXT: v_mul_f32_e32 v2, s0, v2
+; VI-NEXT: v_bfe_u32 v3, v2, 16, 1
+; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v2
+; VI-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; VI-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; VI-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fabs_fold_bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s3, s2, 0x7fff
@@ -351,8 +329,8 @@ define amdgpu_kernel void @fabs_fold_bf16(ptr addrspace(1) %out, bfloat %in0, bf
; GFX9-NEXT: v_mul_f32_e32 v1, s3, v1
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: global_store_short_d16_hi v0, v1, s[0:1]
@@ -360,14 +338,13 @@ define amdgpu_kernel void @fabs_fold_bf16(ptr addrspace(1) %out, bfloat %in0, bf
;
; GFX11-LABEL: fabs_fold_bf16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s1, s0, 0x7fff
-; GFX11-NEXT: s_and_b32 s0, s0, 0xffff0000
-; GFX11-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f32_e64 v0, s1, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_and_b32 s3, s2, 0x7fff
+; GFX11-NEXT: s_and_b32 s2, s2, 0xffff0000
+; GFX11-NEXT: s_lshl_b32 s3, s3, 16
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_f32_e64 v0, s3, s2
; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -376,7 +353,6 @@ define amdgpu_kernel void @fabs_fold_bf16(ptr addrspace(1) %out, bfloat %in0, bf
; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v2
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_store_d16_hi_b16 v3, v0, s[0:1]
; GFX11-NEXT: s_endpgm
%fabs = call bfloat @llvm.fabs.bf16(bfloat %in0)
@@ -455,38 +431,35 @@ define amdgpu_kernel void @v_fabs_v2bf16(ptr addrspace(1) %out, ptr addrspace(1)
define amdgpu_kernel void @fabs_free_v2bf16(ptr addrspace(1) %out, i32 %in) #0 {
; CI-LABEL: fabs_free_v2bf16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0x7fff7fff
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: fabs_free_v2bf16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff7fff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fabs_free_v2bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff7fff
@@ -496,9 +469,7 @@ define amdgpu_kernel void @fabs_free_v2bf16(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX11-LABEL: fabs_free_v2bf16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.f16.ll b/llvm/test/CodeGen/AMDGPU/fabs.f16.ll
index dfd1fa6020eab..3e18936fa7570 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.f16.ll
@@ -12,38 +12,35 @@
define amdgpu_kernel void @s_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
; CI-LABEL: s_fabs_free_f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s2, s2, 0x7fff
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0x7fff
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_short v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_fabs_free_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x7fff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fabs_free_f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff
@@ -53,9 +50,7 @@ define amdgpu_kernel void @s_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
;
; GFX11-TRUE16-LABEL: s_fabs_free_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0x7fff
@@ -66,9 +61,7 @@ define amdgpu_kernel void @s_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
;
; GFX11-FAKE16-LABEL: s_fabs_free_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 0x7fff
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -84,38 +77,35 @@ define amdgpu_kernel void @s_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
define amdgpu_kernel void @s_fabs_f16(ptr addrspace(1) %out, half %in) {
; CI-LABEL: s_fabs_f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s2, s2, 0x7fff
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0x7fff
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_short v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_fabs_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x7fff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fabs_f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff
@@ -125,9 +115,7 @@ define amdgpu_kernel void @s_fabs_f16(ptr addrspace(1) %out, half %in) {
;
; GFX11-TRUE16-LABEL: s_fabs_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0x7fff
@@ -138,9 +126,7 @@ define amdgpu_kernel void @s_fabs_f16(ptr addrspace(1) %out, half %in) {
;
; GFX11-FAKE16-LABEL: s_fabs_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 0x7fff
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -155,38 +141,35 @@ define amdgpu_kernel void @s_fabs_f16(ptr addrspace(1) %out, half %in) {
define amdgpu_kernel void @s_fabs_v2f16(ptr addrspace(1) %out, <2 x half> %in) {
; CI-LABEL: s_fabs_v2f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0x7fff7fff
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_fabs_v2f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff7fff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fabs_v2f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff7fff
@@ -196,9 +179,7 @@ define amdgpu_kernel void @s_fabs_v2f16(ptr addrspace(1) %out, <2 x half> %in) {
;
; GFX11-LABEL: s_fabs_v2f16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -218,13 +199,13 @@ define amdgpu_kernel void @s_fabs_v4f16(ptr addrspace(1) %out, <4 x half> %in) {
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s3, s3, 0x7fff7fff
-; CI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v0, s2
-; CI-NEXT: v_mov_b32_e32 v1, s3
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: s_and_b32 s0, s3, 0x7fff7fff
+; CI-NEXT: s_and_b32 s1, s2, 0x7fff7fff
+; CI-NEXT: v_mov_b32_e32 v2, s1
+; CI-NEXT: v_mov_b32_e32 v3, s0
+; CI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_fabs_v4f16:
@@ -234,13 +215,13 @@ define amdgpu_kernel void @s_fabs_v4f16(ptr addrspace(1) %out, <4 x half> %in) {
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s3, s3, 0x7fff7fff
-; VI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_and_b32 s0, s3, 0x7fff7fff
+; VI-NEXT: s_and_b32 s1, s2, 0x7fff7fff
+; VI-NEXT: v_mov_b32_e32 v2, s1
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fabs_v4f16:
@@ -274,18 +255,16 @@ define amdgpu_kernel void @s_fabs_v4f16(ptr addrspace(1) %out, <4 x half> %in) {
define amdgpu_kernel void @fabs_fold_f16(ptr addrspace(1) %out, half %in0, half %in1) {
; CI-LABEL: fabs_fold_f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[8:9], 0x2
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s1, s0, 16
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s1
-; CI-NEXT: v_cvt_f32_f16_e64 v1, |s0|
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_lshr_b32 s3, s2, 16
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s3
+; CI-NEXT: v_cvt_f32_f16_e64 v1, |s2|
; CI-NEXT: v_mul_f32_e32 v0, v1, v0
; CI-NEXT: v_cvt_f16_f32_e32 v2, v0
-; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_short v[0:1], v2
@@ -293,24 +272,22 @@ define amdgpu_kernel void @fabs_fold_f16(ptr addrspace(1) %out, half %in0, half
;
; VI-LABEL: fabs_fold_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s3, s2, 16
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mul_f16_e64 v2, |s2|, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_lshr_b32 s0, s2, 16
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mul_f16_e64 v2, |s2|, v2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fabs_fold_f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s3, s2, 16
@@ -321,9 +298,7 @@ define amdgpu_kernel void @fabs_fold_f16(ptr addrspace(1) %out, half %in0, half
;
; GFX11-TRUE16-LABEL: fabs_fold_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -334,9 +309,7 @@ define amdgpu_kernel void @fabs_fold_f16(ptr addrspace(1) %out, half %in0, half
;
; GFX11-FAKE16-LABEL: fabs_fold_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -420,38 +393,35 @@ define amdgpu_kernel void @v_fabs_v2f16(ptr addrspace(1) %out, ptr addrspace(1)
define amdgpu_kernel void @fabs_free_v2f16(ptr addrspace(1) %out, i32 %in) #0 {
; CI-LABEL: fabs_free_v2f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0x7fff7fff
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: fabs_free_v2f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff7fff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fabs_free_v2f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff7fff
@@ -461,9 +431,7 @@ define amdgpu_kernel void @fabs_free_v2f16(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX11-LABEL: fabs_free_v2f16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, s2, 0x7fff7fff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.ll b/llvm/test/CodeGen/AMDGPU/fabs.ll
index bb03d62092c32..b49f7b249ce1b 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.ll
@@ -9,25 +9,25 @@
define amdgpu_kernel void @s_fabsf_free(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_fabsf_free:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_bitset0_b32 s4, 31
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_and_b32 s0, s2, 0x7fffffff
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_fabsf_free:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_bitset0_b32 s2, 31
+; VI-NEXT: s_and_b32 s0, s2, 0x7fffffff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%bc= bitcast i32 %in to float
@@ -39,25 +39,25 @@ define amdgpu_kernel void @s_fabsf_free(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @s_fabsf_f32(ptr addrspace(1) %out, float %in) {
; SI-LABEL: s_fabsf_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_bitset0_b32 s4, 31
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_and_b32 s0, s2, 0x7fffffff
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_fabsf_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_bitset0_b32 s2, 31
+; VI-NEXT: s_and_b32 s0, s2, 0x7fffffff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%fabs = call float @llvm.fabs.f32(float %in)
@@ -69,28 +69,29 @@ define amdgpu_kernel void @fabs_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; SI-LABEL: fabs_v2f32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_bitset0_b32 s5, 31
-; SI-NEXT: s_bitset0_b32 s4, 31
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_and_b32 s0, s3, 0x7fffffff
+; SI-NEXT: s_and_b32 s1, s2, 0x7fffffff
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: v_mov_b32_e32 v1, s0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fabs_v2f32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset0_b32 s3, 31
-; VI-NEXT: s_bitset0_b32 s2, 31
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_and_b32 s0, s3, 0x7fffffff
+; VI-NEXT: s_and_b32 s1, s2, 0x7fffffff
+; VI-NEXT: v_mov_b32_e32 v2, s1
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
%fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in)
store <2 x float> %fabs, ptr addrspace(1) %out
@@ -156,10 +157,10 @@ define amdgpu_kernel void @fabs_fold(ptr addrspace(1) %out, float %in0, float %i
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mul_f32_e64 v2, |s2|, v0
+; VI-NEXT: v_mov_b32_e32 v2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mul_f32_e64 v2, |s2|, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%fabs = call float @llvm.fabs.f32(float %in0)
@@ -171,23 +172,23 @@ define amdgpu_kernel void @fabs_fold(ptr addrspace(1) %out, float %in0, float %i
define amdgpu_kernel void @bitpreserve_fabsf_f32(ptr addrspace(1) %out, float %in) {
; SI-LABEL: bitpreserve_fabsf_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, |s4|, 1.0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, |s2|, 1.0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: bitpreserve_fabsf_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_add_f32_e64 v2, |s2|, 1.0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_add_f32_e64 v2, |s2|, 1.0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%in.bc = bitcast float %in to i32
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
index 43a872787621a..f781f5e79181f 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.f16.ll
@@ -136,19 +136,17 @@ define amdgpu_kernel void @v_test_canonicalize_var_f16(ptr addrspace(1) %out) #1
define amdgpu_kernel void @s_test_canonicalize_var_f16(ptr addrspace(1) %out, i16 zeroext %val.arg) #1 {
; VI-LABEL: s_test_canonicalize_var_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_max_f16_e64 v2, s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_max_f16_e64 v2, s2, s2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_test_canonicalize_var_f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_max_f16_e64 v1, s2, s2
@@ -157,22 +155,18 @@ define amdgpu_kernel void @s_test_canonicalize_var_f16(ptr addrspace(1) %out, i1
;
; CI-LABEL: s_test_canonicalize_var_f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[4:5], 0xb
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s2
; CI-NEXT: s_mov_b32 s2, -1
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: buffer_store_short v0, off, s[0:3], 0
; CI-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: s_test_canonicalize_var_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_max_f16_e64 v0.l, s2, s2
@@ -181,9 +175,7 @@ define amdgpu_kernel void @s_test_canonicalize_var_f16(ptr addrspace(1) %out, i1
;
; GFX11-FAKE16-LABEL: s_test_canonicalize_var_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_max_f16_e64 v1, s2, s2
@@ -1695,23 +1687,21 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_v2f16(ptr addrspace(1) %
define amdgpu_kernel void @s_test_canonicalize_var_v2f16(ptr addrspace(1) %out, i32 zeroext %val.arg) #1 {
; VI-LABEL: s_test_canonicalize_var_v2f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s3, s2, 16
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_max_f16_e64 v0, s2, s2
-; VI-NEXT: v_max_f16_sdwa v1, v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v2, v0, v1
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_lshr_b32 s0, s2, 16
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_max_f16_sdwa v2, v2, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: v_max_f16_e64 v3, s2, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_or_b32_e32 v2, v3, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_test_canonicalize_var_v2f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_pk_max_f16 v1, s2, s2
@@ -1720,27 +1710,23 @@ define amdgpu_kernel void @s_test_canonicalize_var_v2f16(ptr addrspace(1) %out,
;
; CI-LABEL: s_test_canonicalize_var_v2f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[4:5], 0xb
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_lshr_b32 s3, s2, 16
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s3
+; CI-NEXT: v_cvt_f32_f16_e32 v1, s2
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s1, s0, 16
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s1
-; CI-NEXT: v_cvt_f32_f16_e32 v1, s0
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
; CI-NEXT: v_cvt_f16_f32_e32 v1, v1
; CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; CI-NEXT: v_or_b32_e32 v0, v1, v0
-; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; CI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_canonicalize_var_v2f16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_pk_max_f16 v1, s2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
index 6b7ddd9a49087..7e1144d40ccab 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.ll
@@ -132,66 +132,35 @@ define amdgpu_kernel void @v_test_canonicalize_var_f32(ptr addrspace(1) %out) #1
}
define amdgpu_kernel void @s_test_canonicalize_var_f32(ptr addrspace(1) %out, float %val) #1 {
-; GFX6-SDAG-LABEL: s_test_canonicalize_var_f32:
-; GFX6-SDAG: ; %bb.0:
-; GFX6-SDAG-NEXT: s_load_dword s2, s[8:9], 0x2
-; GFX6-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX6-SDAG-NEXT: s_add_i32 s12, s12, s17
-; GFX6-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GFX6-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-SDAG-NEXT: v_mul_f32_e64 v2, 1.0, s2
-; GFX6-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-SDAG-NEXT: flat_store_dword v[0:1], v2
-; GFX6-SDAG-NEXT: s_endpgm
-;
-; GFX6-GISEL-LABEL: s_test_canonicalize_var_f32:
-; GFX6-GISEL: ; %bb.0:
-; GFX6-GISEL-NEXT: s_load_dword s2, s[8:9], 0x2
-; GFX6-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX6-GISEL-NEXT: s_add_i32 s12, s12, s17
-; GFX6-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GFX6-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-GISEL-NEXT: v_mul_f32_e64 v2, 1.0, s2
-; GFX6-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX6-GISEL-NEXT: flat_store_dword v[0:1], v2
-; GFX6-GISEL-NEXT: s_endpgm
-;
-; GFX8-SDAG-LABEL: s_test_canonicalize_var_f32:
-; GFX8-SDAG: ; %bb.0:
-; GFX8-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX8-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX8-SDAG-NEXT: s_add_i32 s12, s12, s17
-; GFX8-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GFX8-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v2, 1.0, s2
-; GFX8-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-SDAG-NEXT: flat_store_dword v[0:1], v2
-; GFX8-SDAG-NEXT: s_endpgm
+; GFX678-SDAG-LABEL: s_test_canonicalize_var_f32:
+; GFX678-SDAG: ; %bb.0:
+; GFX678-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX678-SDAG-NEXT: s_add_i32 s12, s12, s17
+; GFX678-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GFX678-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX678-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX678-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; GFX678-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX678-SDAG-NEXT: v_mul_f32_e64 v2, 1.0, s2
+; GFX678-SDAG-NEXT: flat_store_dword v[0:1], v2
+; GFX678-SDAG-NEXT: s_endpgm
;
-; GFX8-GISEL-LABEL: s_test_canonicalize_var_f32:
-; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX8-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX8-GISEL-NEXT: s_add_i32 s12, s12, s17
-; GFX8-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GFX8-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_mul_f32_e64 v2, 1.0, s2
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-GISEL-NEXT: flat_store_dword v[0:1], v2
-; GFX8-GISEL-NEXT: s_endpgm
+; GFX678-GISEL-LABEL: s_test_canonicalize_var_f32:
+; GFX678-GISEL: ; %bb.0:
+; GFX678-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX678-GISEL-NEXT: s_add_i32 s12, s12, s17
+; GFX678-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GFX678-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX678-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX678-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX678-GISEL-NEXT: v_mul_f32_e64 v2, 1.0, s2
+; GFX678-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX678-GISEL-NEXT: flat_store_dword v[0:1], v2
+; GFX678-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: s_test_canonicalize_var_f32:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_max_f32_e64 v1, s2, s2
@@ -200,8 +169,7 @@ define amdgpu_kernel void @s_test_canonicalize_var_f32(ptr addrspace(1) %out, fl
;
; GFX9-GISEL-LABEL: s_test_canonicalize_var_f32:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-GISEL-NEXT: v_max_f32_e64 v0, s2, s2
@@ -210,9 +178,7 @@ define amdgpu_kernel void @s_test_canonicalize_var_f32(ptr addrspace(1) %out, fl
;
; GFX11-SDAG-LABEL: s_test_canonicalize_var_f32:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_max_f32_e64 v1, s2, s2
@@ -221,9 +187,7 @@ define amdgpu_kernel void @s_test_canonicalize_var_f32(ptr addrspace(1) %out, fl
;
; GFX11-GISEL-LABEL: s_test_canonicalize_var_f32:
; GFX11-GISEL: ; %bb.0:
-; GFX11-GISEL-NEXT: s_clause 0x1
-; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: v_max_f32_e64 v0, s2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll b/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll
index 7b11922e75b4f..37eaf6b11a760 100644
--- a/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcopysign.f32.ll
@@ -24,11 +24,11 @@ define amdgpu_kernel void @s_test_copysign_f32(ptr addrspace(1) %out, float %mag
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_brev_b32 s4, -2
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_bfi_b32 v2, s4, v0, v1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_bfi_b32 v2, s4, v2, v3
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -49,33 +49,31 @@ define amdgpu_kernel void @s_test_copysign_f32(ptr addrspace(1) %out, float %mag
define amdgpu_kernel void @s_test_copysign_f32_0(ptr addrspace(1) %out, float %mag) {
; SI-LABEL: s_test_copysign_f32_0:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0x7fffffff
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0x7fffffff
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_0:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset0_b32 s2, 31
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fffffff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_0:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_bitset0_b32 s2, 31
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -90,33 +88,31 @@ define amdgpu_kernel void @s_test_copysign_f32_0(ptr addrspace(1) %out, float %m
define amdgpu_kernel void @s_test_copysign_f32_1(ptr addrspace(1) %out, float %mag) {
; SI-LABEL: s_test_copysign_f32_1:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0x7fffffff
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0x7fffffff
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_1:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset0_b32 s2, 31
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fffffff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_1:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_bitset0_b32 s2, 31
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -131,33 +127,31 @@ define amdgpu_kernel void @s_test_copysign_f32_1(ptr addrspace(1) %out, float %m
define amdgpu_kernel void @s_test_copysign_f32_10.0(ptr addrspace(1) %out, float %mag) {
; SI-LABEL: s_test_copysign_f32_10.0:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0x7fffffff
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0x7fffffff
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_10.0:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset0_b32 s2, 31
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fffffff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_10.0:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_bitset0_b32 s2, 31
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -172,33 +166,31 @@ define amdgpu_kernel void @s_test_copysign_f32_10.0(ptr addrspace(1) %out, float
define amdgpu_kernel void @s_test_copysign_f32_neg1(ptr addrspace(1) %out, float %mag) {
; SI-LABEL: s_test_copysign_f32_neg1:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_or_b32 s4, s6, 0x80000000
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_or_b32 s0, s2, 0x80000000
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_neg1:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset1_b32 s2, 31
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_or_b32 s0, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_neg1:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_bitset1_b32 s2, 31
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -213,33 +205,31 @@ define amdgpu_kernel void @s_test_copysign_f32_neg1(ptr addrspace(1) %out, float
define amdgpu_kernel void @s_test_copysign_f32_neg10(ptr addrspace(1) %out, float %mag) {
; SI-LABEL: s_test_copysign_f32_neg10:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_or_b32 s4, s6, 0x80000000
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_or_b32 s0, s2, 0x80000000
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_neg10:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset1_b32 s2, 31
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_or_b32 s0, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_neg10:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_bitset1_b32 s2, 31
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -254,33 +244,31 @@ define amdgpu_kernel void @s_test_copysign_f32_neg10(ptr addrspace(1) %out, floa
define amdgpu_kernel void @s_test_copysign_f32_0_mag(ptr addrspace(1) %out, float %sign) {
; SI-LABEL: s_test_copysign_f32_0_mag:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0x80000000
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0x80000000
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_0_mag:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_0_mag:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, s2, 0x80000000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -296,35 +284,33 @@ define amdgpu_kernel void @s_test_copysign_f32_0_mag(ptr addrspace(1) %out, floa
define amdgpu_kernel void @s_test_copysign_f32_1_mag(ptr addrspace(1) %out, float %sign) {
; SI-LABEL: s_test_copysign_f32_1_mag:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0x80000000
-; SI-NEXT: s_or_b32 s4, s4, 1.0
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0x80000000
+; SI-NEXT: s_or_b32 s0, s0, 1.0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_1_mag:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x80000000
-; VI-NEXT: s_or_b32 s2, s2, 1.0
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
+; VI-NEXT: s_or_b32 s0, s0, 1.0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_1_mag:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, s2, 0x80000000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -340,35 +326,33 @@ define amdgpu_kernel void @s_test_copysign_f32_1_mag(ptr addrspace(1) %out, floa
define amdgpu_kernel void @s_test_copysign_f32_10_mag(ptr addrspace(1) %out, float %sign) {
; SI-LABEL: s_test_copysign_f32_10_mag:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0x80000000
-; SI-NEXT: s_or_b32 s4, s4, 0x41200000
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0x80000000
+; SI-NEXT: s_or_b32 s0, s0, 0x41200000
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_10_mag:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x80000000
-; VI-NEXT: s_or_b32 s2, s2, 0x41200000
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
+; VI-NEXT: s_or_b32 s0, s0, 0x41200000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_10_mag:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, s2, 0x80000000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -384,35 +368,33 @@ define amdgpu_kernel void @s_test_copysign_f32_10_mag(ptr addrspace(1) %out, flo
define amdgpu_kernel void @s_test_copysign_f32_neg1_mag(ptr addrspace(1) %out, float %sign) {
; SI-LABEL: s_test_copysign_f32_neg1_mag:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0x80000000
-; SI-NEXT: s_or_b32 s4, s4, 1.0
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0x80000000
+; SI-NEXT: s_or_b32 s0, s0, 1.0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_neg1_mag:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x80000000
-; VI-NEXT: s_or_b32 s2, s2, 1.0
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
+; VI-NEXT: s_or_b32 s0, s0, 1.0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_neg1_mag:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, s2, 0x80000000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -428,35 +410,33 @@ define amdgpu_kernel void @s_test_copysign_f32_neg1_mag(ptr addrspace(1) %out, f
define amdgpu_kernel void @s_test_copysign_f32_neg10_mag(ptr addrspace(1) %out, float %sign) {
; SI-LABEL: s_test_copysign_f32_neg10_mag:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 0x80000000
-; SI-NEXT: s_or_b32 s4, s4, 0x41200000
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 0x80000000
+; SI-NEXT: s_or_b32 s0, s0, 0x41200000
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_neg10_mag:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0x80000000
-; VI-NEXT: s_or_b32 s2, s2, 0x41200000
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x80000000
+; VI-NEXT: s_or_b32 s0, s0, 0x41200000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_neg10_mag:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, s2, 0x80000000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -472,50 +452,52 @@ define amdgpu_kernel void @s_test_copysign_f32_neg10_mag(ptr addrspace(1) %out,
define amdgpu_kernel void @s_test_copysign_v2f32(ptr addrspace(1) %out, <2 x float> %mag, <2 x float> %sign) {
; SI-LABEL: s_test_copysign_v2f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-NEXT: s_brev_b32 s8, -2
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s1
-; SI-NEXT: v_mov_b32_e32 v1, s3
-; SI-NEXT: v_bfi_b32 v1, s8, v0, v1
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v2, s2
-; SI-NEXT: v_bfi_b32 v0, s8, v0, v2
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_brev_b32 s0, -2
+; SI-NEXT: v_mov_b32_e32 v0, s3
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: v_bfi_b32 v1, s0, v0, v1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v2, s8
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_bfi_b32 v0, s0, v0, v2
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_v2f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: s_brev_b32 s6, -2
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s1
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_bfi_b32 v1, s6, v0, v1
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_bfi_b32 v0, s6, v2, v0
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s3
; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_bfi_b32 v3, s6, v2, v3
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v4, s4
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_bfi_b32 v2, s6, v2, v4
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_v2f32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s3
-; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v0, s7
+; GFX11-NEXT: v_mov_b32_e32 v2, s6
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_bfi_b32 v1, 0x7fffffff, s1, v0
-; GFX11-NEXT: v_bfi_b32 v0, 0x7fffffff, s0, v2
-; GFX11-NEXT: global_store_b64 v3, v[0:1], s[4:5]
+; GFX11-NEXT: v_bfi_b32 v1, 0x7fffffff, s3, v0
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fffffff, s2, v2
+; GFX11-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
%result = call <2 x float> @llvm.copysign.v2f32(<2 x float> %mag, <2 x float> %sign)
store <2 x float> %result, ptr addrspace(1) %out, align 8
@@ -877,46 +859,44 @@ define <5 x float> @v_test_copysign_v5f32(<5 x float> %mag, <5 x float> %sign) {
define amdgpu_kernel void @s_test_copysign_f32_fptrunc_f64(ptr addrspace(1) %out, float %mag, double %sign) {
; SI-LABEL: s_test_copysign_f32_fptrunc_f64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_brev_b32 s4, -2
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: v_bfi_b32 v0, s4, v0, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_brev_b32 s0, -2
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_bfi_b32 v0, s0, v0, v1
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_fptrunc_f64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_brev_b32 s0, -2
-; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: s_brev_b32 s3, -2
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s5
+; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_bfi_b32 v2, s0, v0, v1
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_bfi_b32 v2, s3, v2, v3
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_fptrunc_f64:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s7
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_bfi_b32 v0, 0x7fffffff, s0, v0
-; GFX11-NEXT: global_store_b32 v1, v0, s[2:3]
+; GFX11-NEXT: v_bfi_b32 v0, 0x7fffffff, s2, v0
+; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NEXT: s_endpgm
%sign.trunc = fptrunc double %sign to float
%result = call float @llvm.copysign.f32(float %mag, float %sign.trunc)
@@ -988,11 +968,11 @@ define amdgpu_kernel void @s_test_copysign_f32_fpext_f16(ptr addrspace(1) %out,
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_brev_b32 s4, -2
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_lshlrev_b32_e64 v0, 16, s3
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_bfi_b32 v2, s4, v1, v0
+; VI-NEXT: v_lshlrev_b32_e64 v2, 16, s3
+; VI-NEXT: v_mov_b32_e32 v3, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_bfi_b32 v2, s4, v3, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -1015,37 +995,32 @@ define amdgpu_kernel void @s_test_copysign_f32_fpext_f16(ptr addrspace(1) %out,
define amdgpu_kernel void @s_test_copysign_f32_1_fpext_f16(ptr addrspace(1) %out, half %sign) {
; SI-LABEL: s_test_copysign_f32_1_fpext_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_and_b32_e32 v0, 0x80000000, v0
; SI-NEXT: v_or_b32_e32 v0, 1.0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_test_copysign_f32_1_fpext_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b32 s2, s2, 16
-; VI-NEXT: s_and_b32 s2, s2, 0x80000000
-; VI-NEXT: s_or_b32 s2, s2, 1.0
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_lshl_b32 s0, s2, 16
+; VI-NEXT: s_and_b32 s0, s0, 0x80000000
+; VI-NEXT: s_or_b32 s0, s0, 1.0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_copysign_f32_1_fpext_f16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_lshl_b32 s2, s2, 16
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
@@ -1083,11 +1058,11 @@ define amdgpu_kernel void @s_test_copysign_f32_fpext_bf16(ptr addrspace(1) %out,
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_brev_b32 s4, -2
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_lshlrev_b32_e64 v0, 16, s3
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_bfi_b32 v2, s4, v1, v0
+; VI-NEXT: v_lshlrev_b32_e64 v2, 16, s3
+; VI-NEXT: v_mov_b32_e32 v3, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_bfi_b32 v2, s4, v3, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv.ll b/llvm/test/CodeGen/AMDGPU/fdiv.ll
index 73d55bd72a0f9..257757b1eca6f 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv.ll
@@ -96,23 +96,23 @@ define amdgpu_kernel void @s_fdiv_f32_ninf(ptr addrspace(1) %out, float %a, floa
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_div_scale_f32 v1, s[4:5], s3, s3, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_div_scale_f32 v2, vcc, s2, v2, s2
-; GFX8-NEXT: v_rcp_f32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_div_scale_f32 v0, s[4:5], s3, s3, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: v_div_scale_f32 v1, vcc, s2, v1, s2
+; GFX8-NEXT: v_rcp_f32_e32 v3, v0
; GFX8-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX8-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; GFX8-NEXT: v_fma_f32 v4, -v0, v3, 1.0
; GFX8-NEXT: v_fma_f32 v3, v4, v3, v3
-; GFX8-NEXT: v_mul_f32_e32 v4, v2, v3
-; GFX8-NEXT: v_fma_f32 v5, -v1, v4, v2
+; GFX8-NEXT: v_mul_f32_e32 v4, v1, v3
+; GFX8-NEXT: v_fma_f32 v5, -v0, v4, v1
; GFX8-NEXT: v_fma_f32 v4, v5, v3, v4
-; GFX8-NEXT: v_fma_f32 v1, -v1, v4, v2
+; GFX8-NEXT: v_fma_f32 v0, -v0, v4, v1
; GFX8-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX8-NEXT: v_div_fmas_f32 v1, v1, v3, v4
-; GFX8-NEXT: v_div_fixup_f32 v2, v1, s3, v0
+; GFX8-NEXT: v_div_fmas_f32 v3, v0, v3, v4
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_div_fixup_f32 v2, v3, s3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -253,21 +253,21 @@ define amdgpu_kernel void @s_fdiv_f32_ieee(ptr addrspace(1) %out, float %a, floa
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_div_scale_f32 v1, s[4:5], s3, s3, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_div_scale_f32 v2, vcc, s2, v2, s2
-; GFX8-NEXT: v_rcp_f32_e32 v3, v1
-; GFX8-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_div_scale_f32 v0, s[4:5], s3, s3, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: v_div_scale_f32 v1, vcc, s2, v1, s2
+; GFX8-NEXT: v_rcp_f32_e32 v3, v0
+; GFX8-NEXT: v_fma_f32 v4, -v0, v3, 1.0
; GFX8-NEXT: v_fma_f32 v3, v4, v3, v3
-; GFX8-NEXT: v_mul_f32_e32 v4, v2, v3
-; GFX8-NEXT: v_fma_f32 v5, -v1, v4, v2
+; GFX8-NEXT: v_mul_f32_e32 v4, v1, v3
+; GFX8-NEXT: v_fma_f32 v5, -v0, v4, v1
; GFX8-NEXT: v_fma_f32 v4, v5, v3, v4
-; GFX8-NEXT: v_fma_f32 v1, -v1, v4, v2
-; GFX8-NEXT: v_div_fmas_f32 v1, v1, v3, v4
-; GFX8-NEXT: v_div_fixup_f32 v2, v1, s3, v0
+; GFX8-NEXT: v_fma_f32 v0, -v0, v4, v1
+; GFX8-NEXT: v_div_fmas_f32 v3, v0, v3, v4
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_div_fixup_f32 v2, v3, s3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -333,17 +333,18 @@ define amdgpu_kernel void @s_fdiv_25ulp_f32(ptr addrspace(1) %out, float %a, flo
; GFX67-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX67-NEXT: v_mov_b32_e32 v0, 0x6f800000
; GFX67-NEXT: v_mov_b32_e32 v1, 0x2f800000
+; GFX67-NEXT: s_mov_b32 s7, 0xf000
+; GFX67-NEXT: s_mov_b32 s6, -1
; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX67-NEXT: v_cmp_gt_f32_e64 vcc, |s5|, v0
+; GFX67-NEXT: v_cmp_gt_f32_e64 vcc, |s3|, v0
; GFX67-NEXT: v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; GFX67-NEXT: v_mul_f32_e32 v1, s5, v0
+; GFX67-NEXT: v_mul_f32_e32 v1, s3, v0
; GFX67-NEXT: v_rcp_f32_e32 v1, v1
-; GFX67-NEXT: s_mov_b32 s3, 0xf000
-; GFX67-NEXT: s_mov_b32 s2, -1
-; GFX67-NEXT: v_mul_f32_e32 v1, s4, v1
+; GFX67-NEXT: s_mov_b32 s4, s0
+; GFX67-NEXT: s_mov_b32 s5, s1
+; GFX67-NEXT: v_mul_f32_e32 v1, s2, v1
; GFX67-NEXT: v_mul_f32_e32 v0, v0, v1
-; GFX67-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX67-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX67-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_25ulp_f32:
@@ -353,13 +354,13 @@ define amdgpu_kernel void @s_fdiv_25ulp_f32(ptr addrspace(1) %out, float %a, flo
; GFX8-NEXT: v_mov_b32_e32 v1, 0x2f800000
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_cmp_gt_f32_e64 vcc, |s3|, v0
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 1.0, v1, vcc
-; GFX8-NEXT: v_mul_f32_e32 v1, s3, v0
-; GFX8-NEXT: v_rcp_f32_e32 v1, v1
-; GFX8-NEXT: v_mul_f32_e32 v1, s2, v1
-; GFX8-NEXT: v_mul_f32_e32 v2, v0, v1
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 1.0, v1, vcc
+; GFX8-NEXT: v_mul_f32_e32 v0, s3, v2
+; GFX8-NEXT: v_rcp_f32_e32 v3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mul_f32_e32 v3, s2, v3
+; GFX8-NEXT: v_mul_f32_e32 v2, v2, v3
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -440,35 +441,36 @@ define amdgpu_kernel void @s_fdiv_25ulp_ieee_f32(ptr addrspace(1) %out, float %a
; GFX7-LABEL: s_fdiv_25ulp_ieee_f32:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX7-NEXT: v_frexp_mant_f32_e32 v0, s5
+; GFX7-NEXT: v_frexp_mant_f32_e32 v0, s3
; GFX7-NEXT: v_rcp_f32_e32 v0, v0
-; GFX7-NEXT: v_frexp_exp_i32_f32_e32 v1, s5
-; GFX7-NEXT: v_frexp_exp_i32_f32_e32 v2, s4
-; GFX7-NEXT: v_frexp_mant_f32_e32 v3, s4
+; GFX7-NEXT: v_frexp_exp_i32_f32_e32 v1, s3
+; GFX7-NEXT: v_frexp_exp_i32_f32_e32 v2, s2
+; GFX7-NEXT: v_frexp_mant_f32_e32 v3, s2
; GFX7-NEXT: v_mul_f32_e32 v0, v3, v0
; GFX7-NEXT: v_sub_i32_e32 v1, vcc, v2, v1
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
; GFX7-NEXT: v_ldexp_f32_e32 v0, v0, v1
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_25ulp_ieee_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_frexp_mant_f32_e32 v1, s3
-; GFX8-NEXT: v_rcp_f32_e32 v1, v1
-; GFX8-NEXT: v_frexp_exp_i32_f32_e32 v0, s3
-; GFX8-NEXT: v_frexp_exp_i32_f32_e32 v2, s2
-; GFX8-NEXT: v_frexp_mant_f32_e32 v3, s2
-; GFX8-NEXT: v_sub_u32_e32 v0, vcc, v2, v0
-; GFX8-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX8-NEXT: v_ldexp_f32 v2, v1, v0
+; GFX8-NEXT: v_frexp_mant_f32_e32 v2, s3
+; GFX8-NEXT: v_rcp_f32_e32 v2, v2
+; GFX8-NEXT: v_frexp_exp_i32_f32_e32 v3, s3
+; GFX8-NEXT: v_frexp_exp_i32_f32_e32 v4, s2
+; GFX8-NEXT: v_frexp_mant_f32_e32 v5, s2
+; GFX8-NEXT: v_mul_f32_e32 v2, v5, v2
+; GFX8-NEXT: v_sub_u32_e32 v3, vcc, v4, v3
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_ldexp_f32 v2, v2, v3
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -526,23 +528,24 @@ define amdgpu_kernel void @s_fdiv_fast_ieee_f32(ptr addrspace(1) %out, float %a,
; GFX67-LABEL: s_fdiv_fast_ieee_f32:
; GFX67: ; %bb.0: ; %entry
; GFX67-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX67-NEXT: s_mov_b32 s7, 0xf000
+; GFX67-NEXT: s_mov_b32 s6, -1
; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX67-NEXT: v_rcp_f32_e32 v0, s5
-; GFX67-NEXT: s_mov_b32 s3, 0xf000
-; GFX67-NEXT: s_mov_b32 s2, -1
-; GFX67-NEXT: v_mul_f32_e32 v0, s4, v0
-; GFX67-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX67-NEXT: v_rcp_f32_e32 v0, s3
+; GFX67-NEXT: s_mov_b32 s4, s0
+; GFX67-NEXT: s_mov_b32 s5, s1
+; GFX67-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX67-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX67-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_fast_ieee_f32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f32_e32 v0, s3
-; GFX8-NEXT: v_mul_f32_e32 v2, s2, v0
+; GFX8-NEXT: v_rcp_f32_e32 v2, s3
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mul_f32_e32 v2, s2, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -587,23 +590,24 @@ define amdgpu_kernel void @s_fdiv_f32_fast_math(ptr addrspace(1) %out, float %a,
; GFX67-LABEL: s_fdiv_f32_fast_math:
; GFX67: ; %bb.0: ; %entry
; GFX67-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX67-NEXT: s_mov_b32 s7, 0xf000
+; GFX67-NEXT: s_mov_b32 s6, -1
; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX67-NEXT: v_rcp_f32_e32 v0, s5
-; GFX67-NEXT: s_mov_b32 s3, 0xf000
-; GFX67-NEXT: s_mov_b32 s2, -1
-; GFX67-NEXT: v_mul_f32_e32 v0, s4, v0
-; GFX67-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX67-NEXT: v_rcp_f32_e32 v0, s3
+; GFX67-NEXT: s_mov_b32 s4, s0
+; GFX67-NEXT: s_mov_b32 s5, s1
+; GFX67-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX67-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX67-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_f32_fast_math:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f32_e32 v0, s3
-; GFX8-NEXT: v_mul_f32_e32 v2, s2, v0
+; GFX8-NEXT: v_rcp_f32_e32 v2, s3
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mul_f32_e32 v2, s2, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -648,23 +652,24 @@ define amdgpu_kernel void @s_fdiv_ulp25_f32_fast_math(ptr addrspace(1) %out, flo
; GFX67-LABEL: s_fdiv_ulp25_f32_fast_math:
; GFX67: ; %bb.0: ; %entry
; GFX67-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX67-NEXT: s_mov_b32 s7, 0xf000
+; GFX67-NEXT: s_mov_b32 s6, -1
; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX67-NEXT: v_rcp_f32_e32 v0, s5
-; GFX67-NEXT: s_mov_b32 s3, 0xf000
-; GFX67-NEXT: s_mov_b32 s2, -1
-; GFX67-NEXT: v_mul_f32_e32 v0, s4, v0
-; GFX67-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX67-NEXT: v_rcp_f32_e32 v0, s3
+; GFX67-NEXT: s_mov_b32 s4, s0
+; GFX67-NEXT: s_mov_b32 s5, s1
+; GFX67-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX67-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX67-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_ulp25_f32_fast_math:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f32_e32 v0, s3
-; GFX8-NEXT: v_mul_f32_e32 v2, s2, v0
+; GFX8-NEXT: v_rcp_f32_e32 v2, s3
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mul_f32_e32 v2, s2, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -788,23 +793,23 @@ define amdgpu_kernel void @s_fdiv_f32_arcp_daz(ptr addrspace(1) %out, float %a,
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_div_scale_f32 v1, s[4:5], s3, s3, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_div_scale_f32 v2, vcc, s2, v2, s2
-; GFX8-NEXT: v_rcp_f32_e32 v3, v1
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_div_scale_f32 v0, s[4:5], s3, s3, v2
+; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: v_div_scale_f32 v1, vcc, s2, v1, s2
+; GFX8-NEXT: v_rcp_f32_e32 v3, v0
; GFX8-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
-; GFX8-NEXT: v_fma_f32 v4, -v1, v3, 1.0
+; GFX8-NEXT: v_fma_f32 v4, -v0, v3, 1.0
; GFX8-NEXT: v_fma_f32 v3, v4, v3, v3
-; GFX8-NEXT: v_mul_f32_e32 v4, v2, v3
-; GFX8-NEXT: v_fma_f32 v5, -v1, v4, v2
+; GFX8-NEXT: v_mul_f32_e32 v4, v1, v3
+; GFX8-NEXT: v_fma_f32 v5, -v0, v4, v1
; GFX8-NEXT: v_fma_f32 v4, v5, v3, v4
-; GFX8-NEXT: v_fma_f32 v1, -v1, v4, v2
+; GFX8-NEXT: v_fma_f32 v0, -v0, v4, v1
; GFX8-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX8-NEXT: v_div_fmas_f32 v1, v1, v3, v4
-; GFX8-NEXT: v_div_fixup_f32 v2, v1, s3, v0
+; GFX8-NEXT: v_div_fmas_f32 v3, v0, v3, v4
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_div_fixup_f32 v2, v3, s3, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -872,23 +877,24 @@ define amdgpu_kernel void @s_fdiv_f32_arcp_ninf(ptr addrspace(1) %out, float %a,
; GFX67-LABEL: s_fdiv_f32_arcp_ninf:
; GFX67: ; %bb.0: ; %entry
; GFX67-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX67-NEXT: s_mov_b32 s7, 0xf000
+; GFX67-NEXT: s_mov_b32 s6, -1
; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX67-NEXT: v_rcp_f32_e32 v0, s5
-; GFX67-NEXT: s_mov_b32 s3, 0xf000
-; GFX67-NEXT: s_mov_b32 s2, -1
-; GFX67-NEXT: v_mul_f32_e32 v0, s4, v0
-; GFX67-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX67-NEXT: v_rcp_f32_e32 v0, s3
+; GFX67-NEXT: s_mov_b32 s4, s0
+; GFX67-NEXT: s_mov_b32 s5, s1
+; GFX67-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX67-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX67-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_f32_arcp_ninf:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f32_e32 v0, s3
-; GFX8-NEXT: v_mul_f32_e32 v2, s2, v0
+; GFX8-NEXT: v_rcp_f32_e32 v2, s3
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mul_f32_e32 v2, s2, v2
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -932,16 +938,18 @@ entry:
define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <2 x float> %b) #0 {
; GFX6-FASTFMA-LABEL: s_fdiv_v2f32:
; GFX6-FASTFMA: ; %bb.0: ; %entry
-; GFX6-FASTFMA-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xb
-; GFX6-FASTFMA-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-FASTFMA-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-FASTFMA-NEXT: s_mov_b32 s2, -1
+; GFX6-FASTFMA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-FASTFMA-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX6-FASTFMA-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-FASTFMA-NEXT: s_mov_b32 s6, -1
; GFX6-FASTFMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-FASTFMA-NEXT: v_mov_b32_e32 v0, s9
-; GFX6-FASTFMA-NEXT: v_div_scale_f32 v1, s[4:5], s11, s11, v0
+; GFX6-FASTFMA-NEXT: v_mov_b32_e32 v0, s3
+; GFX6-FASTFMA-NEXT: s_mov_b32 s4, s0
+; GFX6-FASTFMA-NEXT: s_mov_b32 s5, s1
+; GFX6-FASTFMA-NEXT: v_div_scale_f32 v1, s[0:1], s9, s9, v0
; GFX6-FASTFMA-NEXT: v_rcp_f32_e32 v2, v1
-; GFX6-FASTFMA-NEXT: v_mov_b32_e32 v3, s11
-; GFX6-FASTFMA-NEXT: v_div_scale_f32 v3, vcc, s9, v3, s9
+; GFX6-FASTFMA-NEXT: v_mov_b32_e32 v3, s9
+; GFX6-FASTFMA-NEXT: v_div_scale_f32 v3, vcc, s3, v3, s3
; GFX6-FASTFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FASTFMA-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX6-FASTFMA-NEXT: v_fma_f32 v2, v4, v2, v2
@@ -951,12 +959,12 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX6-FASTFMA-NEXT: v_fma_f32 v1, -v1, v4, v3
; GFX6-FASTFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX6-FASTFMA-NEXT: v_div_fmas_f32 v1, v1, v2, v4
-; GFX6-FASTFMA-NEXT: v_div_fixup_f32 v1, v1, s11, v0
-; GFX6-FASTFMA-NEXT: v_mov_b32_e32 v0, s8
-; GFX6-FASTFMA-NEXT: v_div_scale_f32 v2, s[4:5], s10, s10, v0
+; GFX6-FASTFMA-NEXT: v_div_fixup_f32 v1, v1, s9, v0
+; GFX6-FASTFMA-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-FASTFMA-NEXT: v_div_scale_f32 v2, s[0:1], s8, s8, v0
; GFX6-FASTFMA-NEXT: v_rcp_f32_e32 v3, v2
-; GFX6-FASTFMA-NEXT: v_mov_b32_e32 v4, s10
-; GFX6-FASTFMA-NEXT: v_div_scale_f32 v4, vcc, s8, v4, s8
+; GFX6-FASTFMA-NEXT: v_mov_b32_e32 v4, s8
+; GFX6-FASTFMA-NEXT: v_div_scale_f32 v4, vcc, s2, v4, s2
; GFX6-FASTFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-FASTFMA-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX6-FASTFMA-NEXT: v_fma_f32 v3, v5, v3, v3
@@ -966,20 +974,21 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX6-FASTFMA-NEXT: v_fma_f32 v2, -v2, v5, v4
; GFX6-FASTFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX6-FASTFMA-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX6-FASTFMA-NEXT: v_div_fixup_f32 v0, v2, s10, v0
-; GFX6-FASTFMA-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX6-FASTFMA-NEXT: v_div_fixup_f32 v0, v2, s8, v0
+; GFX6-FASTFMA-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-FASTFMA-NEXT: s_endpgm
;
; GFX6-SLOWFMA-LABEL: s_fdiv_v2f32:
; GFX6-SLOWFMA: ; %bb.0: ; %entry
-; GFX6-SLOWFMA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX6-SLOWFMA-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX6-SLOWFMA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-SLOWFMA-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; GFX6-SLOWFMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-SLOWFMA-NEXT: v_mov_b32_e32 v0, s1
-; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v1, s[6:7], s3, s3, v0
-; GFX6-SLOWFMA-NEXT: v_mov_b32_e32 v2, s3
-; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v2, vcc, s1, v2, s1
-; GFX6-SLOWFMA-NEXT: v_mov_b32_e32 v4, s0
+; GFX6-SLOWFMA-NEXT: v_mov_b32_e32 v0, s3
+; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v1, s[6:7], s5, s5, v0
+; GFX6-SLOWFMA-NEXT: v_mov_b32_e32 v2, s5
+; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v2, vcc, s3, v2, s3
+; GFX6-SLOWFMA-NEXT: v_mov_b32_e32 v4, s2
+; GFX6-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000
; GFX6-SLOWFMA-NEXT: v_rcp_f32_e32 v3, v1
; GFX6-SLOWFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-SLOWFMA-NEXT: v_fma_f32 v5, -v1, v3, 1.0
@@ -989,14 +998,13 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX6-SLOWFMA-NEXT: v_fma_f32 v5, v6, v3, v5
; GFX6-SLOWFMA-NEXT: v_fma_f32 v1, -v1, v5, v2
; GFX6-SLOWFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v2, s[6:7], s2, s2, v4
+; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v2, s[6:7], s4, s4, v4
; GFX6-SLOWFMA-NEXT: v_div_fmas_f32 v1, v1, v3, v5
-; GFX6-SLOWFMA-NEXT: v_mov_b32_e32 v3, s2
-; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v3, vcc, s0, v3, s0
-; GFX6-SLOWFMA-NEXT: s_mov_b32 s7, 0xf000
-; GFX6-SLOWFMA-NEXT: s_mov_b32 s6, -1
+; GFX6-SLOWFMA-NEXT: v_mov_b32_e32 v3, s4
+; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v3, vcc, s2, v3, s2
+; GFX6-SLOWFMA-NEXT: s_mov_b32 s2, -1
; GFX6-SLOWFMA-NEXT: v_rcp_f32_e32 v5, v2
-; GFX6-SLOWFMA-NEXT: v_div_fixup_f32 v1, v1, s3, v0
+; GFX6-SLOWFMA-NEXT: v_div_fixup_f32 v1, v1, s5, v0
; GFX6-SLOWFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-SLOWFMA-NEXT: v_fma_f32 v0, -v2, v5, 1.0
; GFX6-SLOWFMA-NEXT: v_fma_f32 v0, v0, v5, v5
@@ -1006,22 +1014,24 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX6-SLOWFMA-NEXT: v_fma_f32 v2, -v2, v5, v3
; GFX6-SLOWFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX6-SLOWFMA-NEXT: v_div_fmas_f32 v0, v2, v0, v5
-; GFX6-SLOWFMA-NEXT: v_div_fixup_f32 v0, v0, s2, v4
-; GFX6-SLOWFMA-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX6-SLOWFMA-NEXT: v_div_fixup_f32 v0, v0, s4, v4
+; GFX6-SLOWFMA-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX6-SLOWFMA-NEXT: s_endpgm
;
; GFX7-LABEL: s_fdiv_v2f32:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v0, s9
-; GFX7-NEXT: v_div_scale_f32 v1, s[4:5], s11, s11, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s3
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_div_scale_f32 v1, s[0:1], s9, s9, v0
; GFX7-NEXT: v_rcp_f32_e32 v2, v1
-; GFX7-NEXT: v_mov_b32_e32 v3, s11
-; GFX7-NEXT: v_div_scale_f32 v3, vcc, s9, v3, s9
+; GFX7-NEXT: v_mov_b32_e32 v3, s9
+; GFX7-NEXT: v_div_scale_f32 v3, vcc, s3, v3, s3
; GFX7-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX7-NEXT: v_fma_f32 v4, -v1, v2, 1.0
; GFX7-NEXT: v_fma_f32 v2, v4, v2, v2
@@ -1031,12 +1041,12 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX7-NEXT: v_fma_f32 v1, -v1, v4, v3
; GFX7-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX7-NEXT: v_div_fmas_f32 v1, v1, v2, v4
-; GFX7-NEXT: v_div_fixup_f32 v1, v1, s11, v0
-; GFX7-NEXT: v_mov_b32_e32 v0, s8
-; GFX7-NEXT: v_div_scale_f32 v2, s[4:5], s10, s10, v0
+; GFX7-NEXT: v_div_fixup_f32 v1, v1, s9, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, s2
+; GFX7-NEXT: v_div_scale_f32 v2, s[0:1], s8, s8, v0
; GFX7-NEXT: v_rcp_f32_e32 v3, v2
-; GFX7-NEXT: v_mov_b32_e32 v4, s10
-; GFX7-NEXT: v_div_scale_f32 v4, vcc, s8, v4, s8
+; GFX7-NEXT: v_mov_b32_e32 v4, s8
+; GFX7-NEXT: v_div_scale_f32 v4, vcc, s2, v4, s2
; GFX7-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX7-NEXT: v_fma_f32 v5, -v2, v3, 1.0
; GFX7-NEXT: v_fma_f32 v3, v5, v3, v3
@@ -1046,19 +1056,20 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX7-NEXT: v_fma_f32 v2, -v2, v5, v4
; GFX7-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX7-NEXT: v_div_fmas_f32 v2, v2, v3, v5
-; GFX7-NEXT: v_div_fixup_f32 v0, v2, s10, v0
-; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX7-NEXT: v_div_fixup_f32 v0, v2, s8, v0
+; GFX7-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_v2f32:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s1
-; GFX8-NEXT: v_div_scale_f32 v1, s[6:7], s3, s3, v0
-; GFX8-NEXT: v_mov_b32_e32 v2, s3
-; GFX8-NEXT: v_div_scale_f32 v2, vcc, s1, v2, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s0
+; GFX8-NEXT: v_mov_b32_e32 v0, s3
+; GFX8-NEXT: v_div_scale_f32 v1, s[6:7], s5, s5, v0
+; GFX8-NEXT: v_mov_b32_e32 v2, s5
+; GFX8-NEXT: v_div_scale_f32 v2, vcc, s3, v2, s3
+; GFX8-NEXT: v_mov_b32_e32 v4, s2
; GFX8-NEXT: v_rcp_f32_e32 v3, v1
; GFX8-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX8-NEXT: v_fma_f32 v5, -v1, v3, 1.0
@@ -1068,13 +1079,12 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX8-NEXT: v_fma_f32 v5, v6, v3, v5
; GFX8-NEXT: v_fma_f32 v1, -v1, v5, v2
; GFX8-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX8-NEXT: v_div_scale_f32 v2, s[6:7], s2, s2, v4
+; GFX8-NEXT: v_div_scale_f32 v2, s[6:7], s4, s4, v4
; GFX8-NEXT: v_div_fmas_f32 v1, v1, v3, v5
-; GFX8-NEXT: v_mov_b32_e32 v3, s2
-; GFX8-NEXT: v_div_scale_f32 v3, vcc, s0, v3, s0
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-NEXT: v_div_scale_f32 v3, vcc, s2, v3, s2
; GFX8-NEXT: v_rcp_f32_e32 v5, v2
-; GFX8-NEXT: v_div_fixup_f32 v1, v1, s3, v0
+; GFX8-NEXT: v_div_fixup_f32 v1, v1, s5, v0
; GFX8-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX8-NEXT: v_fma_f32 v0, -v2, v5, 1.0
; GFX8-NEXT: v_fma_f32 v0, v0, v5, v5
@@ -1084,19 +1094,20 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX8-NEXT: v_fma_f32 v2, -v2, v5, v3
; GFX8-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX8-NEXT: v_div_fmas_f32 v0, v2, v0, v5
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v3, s1
; GFX8-NEXT: v_mov_b32_e32 v2, s0
-; GFX8-NEXT: v_div_fixup_f32 v0, v0, s2, v4
+; GFX8-NEXT: v_mov_b32_e32 v3, s1
+; GFX8-NEXT: v_div_fixup_f32 v0, v0, s4, v4
; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX8-NEXT: s_endpgm
;
; GFX10-LABEL: s_fdiv_v2f32:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_div_scale_f32 v0, s6, s3, s3, s1
-; GFX10-NEXT: v_div_scale_f32 v2, vcc_lo, s1, s3, s1
+; GFX10-NEXT: v_div_scale_f32 v0, s4, s7, s7, s3
+; GFX10-NEXT: v_div_scale_f32 v2, vcc_lo, s3, s7, s3
; GFX10-NEXT: v_rcp_f32_e32 v1, v0
; GFX10-NEXT: s_denorm_mode 15
; GFX10-NEXT: v_fma_f32 v3, -v0, v1, 1.0
@@ -1106,12 +1117,11 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX10-NEXT: v_fmac_f32_e32 v3, v4, v1
; GFX10-NEXT: v_fma_f32 v0, -v0, v3, v2
; GFX10-NEXT: s_denorm_mode 12
-; GFX10-NEXT: v_div_scale_f32 v2, s6, s2, s2, s0
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: v_div_scale_f32 v2, s4, s6, s6, s2
; GFX10-NEXT: v_div_fmas_f32 v0, v0, v1, v3
; GFX10-NEXT: v_rcp_f32_e32 v4, v2
-; GFX10-NEXT: v_div_fixup_f32 v1, v0, s3, s1
-; GFX10-NEXT: v_div_scale_f32 v0, vcc_lo, s0, s2, s0
+; GFX10-NEXT: v_div_fixup_f32 v1, v0, s7, s3
+; GFX10-NEXT: v_div_scale_f32 v0, vcc_lo, s2, s6, s2
; GFX10-NEXT: s_denorm_mode 15
; GFX10-NEXT: v_fma_f32 v3, -v2, v4, 1.0
; GFX10-NEXT: v_fmac_f32_e32 v4, v3, v4
@@ -1122,19 +1132,18 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX10-NEXT: s_denorm_mode 12
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: v_div_fmas_f32 v0, v0, v4, v3
-; GFX10-NEXT: v_div_fixup_f32 v0, v0, s2, s0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX10-NEXT: v_div_fixup_f32 v0, v0, s6, s2
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_v2f32:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_div_scale_f32 v0, null, s3, s3, s1
-; GFX11-NEXT: v_div_scale_f32 v2, vcc_lo, s1, s3, s1
+; GFX11-NEXT: v_div_scale_f32 v0, null, s5, s5, s3
+; GFX11-NEXT: v_div_scale_f32 v2, vcc_lo, s3, s5, s3
; GFX11-NEXT: v_rcp_f32_e32 v1, v0
; GFX11-NEXT: s_denorm_mode 15
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -1145,11 +1154,11 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX11-NEXT: v_fmac_f32_e32 v3, v4, v1
; GFX11-NEXT: v_fma_f32 v0, -v0, v3, v2
; GFX11-NEXT: s_denorm_mode 12
-; GFX11-NEXT: v_div_scale_f32 v2, null, s2, s2, s0
+; GFX11-NEXT: v_div_scale_f32 v2, null, s4, s4, s2
; GFX11-NEXT: v_div_fmas_f32 v0, v0, v1, v3
; GFX11-NEXT: v_rcp_f32_e32 v4, v2
-; GFX11-NEXT: v_div_fixup_f32 v1, v0, s3, s1
-; GFX11-NEXT: v_div_scale_f32 v0, vcc_lo, s0, s2, s0
+; GFX11-NEXT: v_div_fixup_f32 v1, v0, s5, s3
+; GFX11-NEXT: v_div_scale_f32 v0, vcc_lo, s2, s4, s2
; GFX11-NEXT: s_denorm_mode 15
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_fma_f32 v3, -v2, v4, 1.0
@@ -1161,8 +1170,8 @@ define amdgpu_kernel void @s_fdiv_v2f32(ptr addrspace(1) %out, <2 x float> %a, <
; GFX11-NEXT: s_denorm_mode 12
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: v_div_fmas_f32 v0, v0, v4, v3
-; GFX11-NEXT: v_div_fixup_f32 v0, v0, s2, s0
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: v_div_fixup_f32 v0, v0, s4, s2
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; EG-LABEL: s_fdiv_v2f32:
@@ -1187,58 +1196,60 @@ entry:
define amdgpu_kernel void @s_fdiv_ulp25_v2f32(ptr addrspace(1) %out, <2 x float> %a, <2 x float> %b) #0 {
; GFX67-LABEL: s_fdiv_ulp25_v2f32:
; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX67-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX67-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX67-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX67-NEXT: s_mov_b32 s7, 0xf000
; GFX67-NEXT: s_mov_b32 s6, -1
; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: v_rcp_f32_e32 v0, s2
-; GFX67-NEXT: v_rcp_f32_e32 v1, s3
-; GFX67-NEXT: v_mul_f32_e32 v0, s0, v0
-; GFX67-NEXT: v_mul_f32_e32 v1, s1, v1
+; GFX67-NEXT: v_rcp_f32_e32 v0, s8
+; GFX67-NEXT: v_rcp_f32_e32 v1, s9
+; GFX67-NEXT: s_mov_b32 s4, s0
+; GFX67-NEXT: s_mov_b32 s5, s1
+; GFX67-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX67-NEXT: v_mul_f32_e32 v1, s3, v1
; GFX67-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX67-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_ulp25_v2f32:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f32_e32 v0, s2
-; GFX8-NEXT: v_rcp_f32_e32 v1, s3
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
-; GFX8-NEXT: v_mov_b32_e32 v3, s5
-; GFX8-NEXT: v_mul_f32_e32 v0, s0, v0
-; GFX8-NEXT: v_mul_f32_e32 v1, s1, v1
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_rcp_f32_e32 v2, s6
+; GFX8-NEXT: v_rcp_f32_e32 v3, s7
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mul_f32_e32 v2, s2, v2
+; GFX8-NEXT: v_mul_f32_e32 v3, s3, v3
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX10-LABEL: s_fdiv_ulp25_v2f32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f32_e32 v0, s2
-; GFX10-NEXT: v_rcp_f32_e32 v1, s3
-; GFX10-NEXT: v_mul_f32_e32 v0, s0, v0
-; GFX10-NEXT: v_mul_f32_e32 v1, s1, v1
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX10-NEXT: v_rcp_f32_e32 v0, s6
+; GFX10-NEXT: v_rcp_f32_e32 v1, s7
+; GFX10-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX10-NEXT: v_mul_f32_e32 v1, s3, v1
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_ulp25_v2f32:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_rcp_f32_e32 v0, s2
-; GFX11-NEXT: v_rcp_f32_e32 v1, s3
+; GFX11-NEXT: v_rcp_f32_e32 v0, s6
+; GFX11-NEXT: v_rcp_f32_e32 v1, s7
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_dual_mul_f32 v0, s0, v0 :: v_dual_mul_f32 v1, s1, v1
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: v_dual_mul_f32 v0, s2, v0 :: v_dual_mul_f32 v1, s3, v1
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; EG-LABEL: s_fdiv_ulp25_v2f32:
@@ -1263,58 +1274,60 @@ entry:
define amdgpu_kernel void @s_fdiv_v2f32_fast_math(ptr addrspace(1) %out, <2 x float> %a, <2 x float> %b) #0 {
; GFX67-LABEL: s_fdiv_v2f32_fast_math:
; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX67-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX67-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX67-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX67-NEXT: s_mov_b32 s7, 0xf000
; GFX67-NEXT: s_mov_b32 s6, -1
; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: v_rcp_f32_e32 v0, s3
-; GFX67-NEXT: v_rcp_f32_e32 v2, s2
-; GFX67-NEXT: v_mul_f32_e32 v1, s1, v0
-; GFX67-NEXT: v_mul_f32_e32 v0, s0, v2
+; GFX67-NEXT: v_rcp_f32_e32 v0, s9
+; GFX67-NEXT: v_rcp_f32_e32 v2, s8
+; GFX67-NEXT: s_mov_b32 s4, s0
+; GFX67-NEXT: s_mov_b32 s5, s1
+; GFX67-NEXT: v_mul_f32_e32 v1, s3, v0
+; GFX67-NEXT: v_mul_f32_e32 v0, s2, v2
; GFX67-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX67-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_v2f32_fast_math:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f32_e32 v0, s3
-; GFX8-NEXT: v_rcp_f32_e32 v2, s2
-; GFX8-NEXT: v_mul_f32_e32 v1, s1, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, s0, v2
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
-; GFX8-NEXT: v_mov_b32_e32 v3, s5
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_rcp_f32_e32 v2, s7
+; GFX8-NEXT: v_rcp_f32_e32 v4, s6
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mul_f32_e32 v3, s3, v2
+; GFX8-NEXT: v_mul_f32_e32 v2, s2, v4
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX10-LABEL: s_fdiv_v2f32_fast_math:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f32_e32 v0, s3
-; GFX10-NEXT: v_rcp_f32_e32 v2, s2
-; GFX10-NEXT: v_mul_f32_e32 v1, s1, v0
-; GFX10-NEXT: v_mul_f32_e32 v0, s0, v2
-; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[6:7]
+; GFX10-NEXT: v_rcp_f32_e32 v0, s7
+; GFX10-NEXT: v_rcp_f32_e32 v2, s6
+; GFX10-NEXT: v_mul_f32_e32 v1, s3, v0
+; GFX10-NEXT: v_mul_f32_e32 v0, s2, v2
+; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_v2f32_fast_math:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_rcp_f32_e32 v0, s3
-; GFX11-NEXT: v_rcp_f32_e32 v2, s2
+; GFX11-NEXT: v_rcp_f32_e32 v0, s7
+; GFX11-NEXT: v_rcp_f32_e32 v2, s6
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_dual_mul_f32 v1, s1, v0 :: v_dual_mul_f32 v0, s0, v2
-; GFX11-NEXT: global_store_b64 v3, v[0:1], s[4:5]
+; GFX11-NEXT: v_dual_mul_f32 v1, s3, v0 :: v_dual_mul_f32 v0, s2, v2
+; GFX11-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; EG-LABEL: s_fdiv_v2f32_fast_math:
@@ -1339,58 +1352,60 @@ entry:
define amdgpu_kernel void @s_fdiv_v2f32_arcp_math(ptr addrspace(1) %out, <2 x float> %a, <2 x float> %b) #0 {
; GFX67-LABEL: s_fdiv_v2f32_arcp_math:
; GFX67: ; %bb.0: ; %entry
-; GFX67-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX67-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX67-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GFX67-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX67-NEXT: s_mov_b32 s7, 0xf000
; GFX67-NEXT: s_mov_b32 s6, -1
; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: v_rcp_f32_e32 v0, s3
-; GFX67-NEXT: v_rcp_f32_e32 v2, s2
-; GFX67-NEXT: v_mul_f32_e32 v1, s1, v0
-; GFX67-NEXT: v_mul_f32_e32 v0, s0, v2
+; GFX67-NEXT: v_rcp_f32_e32 v0, s9
+; GFX67-NEXT: v_rcp_f32_e32 v2, s8
+; GFX67-NEXT: s_mov_b32 s4, s0
+; GFX67-NEXT: s_mov_b32 s5, s1
+; GFX67-NEXT: v_mul_f32_e32 v1, s3, v0
+; GFX67-NEXT: v_mul_f32_e32 v0, s2, v2
; GFX67-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX67-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_v2f32_arcp_math:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_rcp_f32_e32 v0, s3
-; GFX8-NEXT: v_rcp_f32_e32 v2, s2
-; GFX8-NEXT: v_mul_f32_e32 v1, s1, v0
-; GFX8-NEXT: v_mul_f32_e32 v0, s0, v2
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
-; GFX8-NEXT: v_mov_b32_e32 v3, s5
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_rcp_f32_e32 v2, s7
+; GFX8-NEXT: v_rcp_f32_e32 v4, s6
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mul_f32_e32 v3, s3, v2
+; GFX8-NEXT: v_mul_f32_e32 v2, s2, v4
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX10-LABEL: s_fdiv_v2f32_arcp_math:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_rcp_f32_e32 v0, s3
-; GFX10-NEXT: v_rcp_f32_e32 v2, s2
-; GFX10-NEXT: v_mul_f32_e32 v1, s1, v0
-; GFX10-NEXT: v_mul_f32_e32 v0, s0, v2
-; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[6:7]
+; GFX10-NEXT: v_rcp_f32_e32 v0, s7
+; GFX10-NEXT: v_rcp_f32_e32 v2, s6
+; GFX10-NEXT: v_mul_f32_e32 v1, s3, v0
+; GFX10-NEXT: v_mul_f32_e32 v0, s2, v2
+; GFX10-NEXT: global_store_dwordx2 v3, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_v2f32_arcp_math:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_rcp_f32_e32 v0, s3
-; GFX11-NEXT: v_rcp_f32_e32 v2, s2
+; GFX11-NEXT: v_rcp_f32_e32 v0, s7
+; GFX11-NEXT: v_rcp_f32_e32 v2, s6
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
-; GFX11-NEXT: v_dual_mul_f32 v1, s1, v0 :: v_dual_mul_f32 v0, s0, v2
-; GFX11-NEXT: global_store_b64 v3, v[0:1], s[4:5]
+; GFX11-NEXT: v_dual_mul_f32 v1, s3, v0 :: v_dual_mul_f32 v0, s2, v2
+; GFX11-NEXT: global_store_b64 v3, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; EG-LABEL: s_fdiv_v2f32_arcp_math:
@@ -2072,15 +2087,15 @@ define amdgpu_kernel void @s_fdiv_v4f32_arcp_math(ptr addrspace(1) %out, ptr add
define amdgpu_kernel void @s_fdiv_f32_correctly_rounded_divide_sqrt(ptr addrspace(1) %out, float %a) #0 {
; GFX6-FASTFMA-LABEL: s_fdiv_f32_correctly_rounded_divide_sqrt:
; GFX6-FASTFMA: ; %bb.0: ; %entry
-; GFX6-FASTFMA-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-FASTFMA-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-FASTFMA-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-FASTFMA-NEXT: s_mov_b32 s2, -1
+; GFX6-FASTFMA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-FASTFMA-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-FASTFMA-NEXT: s_mov_b32 s6, -1
; GFX6-FASTFMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-FASTFMA-NEXT: v_div_scale_f32 v0, s[4:5], s6, s6, 1.0
+; GFX6-FASTFMA-NEXT: v_div_scale_f32 v0, s[8:9], s2, s2, 1.0
; GFX6-FASTFMA-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-FASTFMA-NEXT: v_div_scale_f32 v2, vcc, 1.0, s6, 1.0
+; GFX6-FASTFMA-NEXT: v_div_scale_f32 v2, vcc, 1.0, s2, 1.0
; GFX6-FASTFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX6-FASTFMA-NEXT: s_mov_b32 s4, s0
; GFX6-FASTFMA-NEXT: v_fma_f32 v3, -v0, v1, 1.0
; GFX6-FASTFMA-NEXT: v_fma_f32 v1, v3, v1, v1
; GFX6-FASTFMA-NEXT: v_mul_f32_e32 v3, v2, v1
@@ -2089,19 +2104,21 @@ define amdgpu_kernel void @s_fdiv_f32_correctly_rounded_divide_sqrt(ptr addrspac
; GFX6-FASTFMA-NEXT: v_fma_f32 v0, -v0, v3, v2
; GFX6-FASTFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX6-FASTFMA-NEXT: v_div_fmas_f32 v0, v0, v1, v3
-; GFX6-FASTFMA-NEXT: v_div_fixup_f32 v0, v0, s6, 1.0
-; GFX6-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-FASTFMA-NEXT: s_mov_b32 s5, s1
+; GFX6-FASTFMA-NEXT: v_div_fixup_f32 v0, v0, s2, 1.0
+; GFX6-FASTFMA-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-FASTFMA-NEXT: s_endpgm
;
; GFX6-SLOWFMA-LABEL: s_fdiv_f32_correctly_rounded_divide_sqrt:
; GFX6-SLOWFMA: ; %bb.0: ; %entry
-; GFX6-SLOWFMA-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-SLOWFMA-NEXT: s_mov_b32 s2, -1
+; GFX6-SLOWFMA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-SLOWFMA-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SLOWFMA-NEXT: s_mov_b32 s6, -1
; GFX6-SLOWFMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v0, s[0:1], s6, s6, 1.0
-; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v1, vcc, 1.0, s6, 1.0
-; GFX6-SLOWFMA-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v0, s[4:5], s2, s2, 1.0
+; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v1, vcc, 1.0, s2, 1.0
+; GFX6-SLOWFMA-NEXT: s_mov_b32 s4, s0
+; GFX6-SLOWFMA-NEXT: s_mov_b32 s5, s1
; GFX6-SLOWFMA-NEXT: v_rcp_f32_e32 v2, v0
; GFX6-SLOWFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX6-SLOWFMA-NEXT: v_fma_f32 v3, -v0, v2, 1.0
@@ -2112,22 +2129,21 @@ define amdgpu_kernel void @s_fdiv_f32_correctly_rounded_divide_sqrt(ptr addrspac
; GFX6-SLOWFMA-NEXT: v_fma_f32 v0, -v0, v3, v1
; GFX6-SLOWFMA-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX6-SLOWFMA-NEXT: v_div_fmas_f32 v0, v0, v2, v3
-; GFX6-SLOWFMA-NEXT: v_div_fixup_f32 v0, v0, s6, 1.0
-; GFX6-SLOWFMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-SLOWFMA-NEXT: v_div_fixup_f32 v0, v0, s2, 1.0
+; GFX6-SLOWFMA-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-SLOWFMA-NEXT: s_endpgm
;
; GFX7-LABEL: s_fdiv_f32_correctly_rounded_divide_sqrt:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_div_scale_f32 v0, s[4:5], s6, s6, 1.0
+; GFX7-NEXT: v_div_scale_f32 v0, s[8:9], s2, s2, 1.0
; GFX7-NEXT: v_rcp_f32_e32 v1, v0
-; GFX7-NEXT: v_div_scale_f32 v2, vcc, 1.0, s6, 1.0
+; GFX7-NEXT: v_div_scale_f32 v2, vcc, 1.0, s2, 1.0
; GFX7-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
+; GFX7-NEXT: s_mov_b32 s4, s0
; GFX7-NEXT: v_fma_f32 v3, -v0, v1, 1.0
; GFX7-NEXT: v_fma_f32 v1, v3, v1, v1
; GFX7-NEXT: v_mul_f32_e32 v3, v2, v1
@@ -2136,17 +2152,17 @@ define amdgpu_kernel void @s_fdiv_f32_correctly_rounded_divide_sqrt(ptr addrspac
; GFX7-NEXT: v_fma_f32 v0, -v0, v3, v2
; GFX7-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
; GFX7-NEXT: v_div_fmas_f32 v0, v0, v1, v3
-; GFX7-NEXT: v_div_fixup_f32 v0, v0, s6, 1.0
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: s_mov_b32 s5, s1
+; GFX7-NEXT: v_div_fixup_f32 v0, v0, s2, 1.0
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_f32_correctly_rounded_divide_sqrt:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_div_scale_f32 v0, s[0:1], s2, s2, 1.0
+; GFX8-NEXT: v_div_scale_f32 v0, s[4:5], s2, s2, 1.0
; GFX8-NEXT: v_div_scale_f32 v1, vcc, 1.0, s2, 1.0
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: v_rcp_f32_e32 v2, v0
; GFX8-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 3
; GFX8-NEXT: v_fma_f32 v3, -v0, v2, 1.0
@@ -2156,21 +2172,19 @@ define amdgpu_kernel void @s_fdiv_f32_correctly_rounded_divide_sqrt(ptr addrspac
; GFX8-NEXT: v_fma_f32 v3, v4, v2, v3
; GFX8-NEXT: v_fma_f32 v0, -v0, v3, v1
; GFX8-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_MODE, 4, 2), 0
-; GFX8-NEXT: v_div_fmas_f32 v0, v0, v2, v3
-; GFX8-NEXT: v_div_fixup_f32 v2, v0, s2, 1.0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_div_fmas_f32 v2, v0, v2, v3
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_div_fixup_f32 v2, v2, s2, 1.0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX10-LABEL: s_fdiv_f32_correctly_rounded_divide_sqrt:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_div_scale_f32 v0, s0, s2, s2, 1.0
+; GFX10-NEXT: v_div_scale_f32 v0, s3, s2, s2, 1.0
; GFX10-NEXT: v_div_scale_f32 v2, vcc_lo, 1.0, s2, 1.0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX10-NEXT: v_rcp_f32_e32 v1, v0
; GFX10-NEXT: s_denorm_mode 15
; GFX10-NEXT: v_fma_f32 v3, -v0, v1, 1.0
@@ -2183,15 +2197,12 @@ define amdgpu_kernel void @s_fdiv_f32_correctly_rounded_divide_sqrt(ptr addrspac
; GFX10-NEXT: v_div_fmas_f32 v0, v0, v1, v3
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_div_fixup_f32 v0, v0, s2, 1.0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_f32_correctly_rounded_divide_sqrt:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_div_scale_f32 v0, null, s2, s2, 1.0
; GFX11-NEXT: v_div_scale_f32 v2, vcc_lo, 1.0, s2, 1.0
@@ -2230,14 +2241,15 @@ entry:
define amdgpu_kernel void @s_fdiv_f32_denorms_correctly_rounded_divide_sqrt(ptr addrspace(1) %out, float %a) #1 {
; GFX6-FASTFMA-LABEL: s_fdiv_f32_denorms_correctly_rounded_divide_sqrt:
; GFX6-FASTFMA: ; %bb.0: ; %entry
-; GFX6-FASTFMA-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-FASTFMA-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-FASTFMA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-FASTFMA-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-FASTFMA-NEXT: s_mov_b32 s6, -1
; GFX6-FASTFMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-FASTFMA-NEXT: v_div_scale_f32 v0, s[2:3], s6, s6, 1.0
+; GFX6-FASTFMA-NEXT: v_div_scale_f32 v0, s[4:5], s2, s2, 1.0
; GFX6-FASTFMA-NEXT: v_rcp_f32_e32 v1, v0
-; GFX6-FASTFMA-NEXT: v_div_scale_f32 v2, vcc, 1.0, s6, 1.0
-; GFX6-FASTFMA-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-FASTFMA-NEXT: s_mov_b32 s2, -1
+; GFX6-FASTFMA-NEXT: v_div_scale_f32 v2, vcc, 1.0, s2, 1.0
+; GFX6-FASTFMA-NEXT: s_mov_b32 s4, s0
+; GFX6-FASTFMA-NEXT: s_mov_b32 s5, s1
; GFX6-FASTFMA-NEXT: v_fma_f32 v3, -v0, v1, 1.0
; GFX6-FASTFMA-NEXT: v_fma_f32 v1, v3, v1, v1
; GFX6-FASTFMA-NEXT: v_mul_f32_e32 v3, v2, v1
@@ -2245,19 +2257,20 @@ define amdgpu_kernel void @s_fdiv_f32_denorms_correctly_rounded_divide_sqrt(ptr
; GFX6-FASTFMA-NEXT: v_fma_f32 v3, v4, v1, v3
; GFX6-FASTFMA-NEXT: v_fma_f32 v0, -v0, v3, v2
; GFX6-FASTFMA-NEXT: v_div_fmas_f32 v0, v0, v1, v3
-; GFX6-FASTFMA-NEXT: v_div_fixup_f32 v0, v0, s6, 1.0
-; GFX6-FASTFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-FASTFMA-NEXT: v_div_fixup_f32 v0, v0, s2, 1.0
+; GFX6-FASTFMA-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-FASTFMA-NEXT: s_endpgm
;
; GFX6-SLOWFMA-LABEL: s_fdiv_f32_denorms_correctly_rounded_divide_sqrt:
; GFX6-SLOWFMA: ; %bb.0: ; %entry
-; GFX6-SLOWFMA-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-SLOWFMA-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-SLOWFMA-NEXT: s_mov_b32 s2, -1
+; GFX6-SLOWFMA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-SLOWFMA-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SLOWFMA-NEXT: s_mov_b32 s6, -1
; GFX6-SLOWFMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v0, s[0:1], s6, s6, 1.0
-; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v1, vcc, 1.0, s6, 1.0
-; GFX6-SLOWFMA-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v0, s[4:5], s2, s2, 1.0
+; GFX6-SLOWFMA-NEXT: v_div_scale_f32 v1, vcc, 1.0, s2, 1.0
+; GFX6-SLOWFMA-NEXT: s_mov_b32 s4, s0
+; GFX6-SLOWFMA-NEXT: s_mov_b32 s5, s1
; GFX6-SLOWFMA-NEXT: v_rcp_f32_e32 v2, v0
; GFX6-SLOWFMA-NEXT: v_fma_f32 v3, -v0, v2, 1.0
; GFX6-SLOWFMA-NEXT: v_fma_f32 v2, v3, v2, v2
@@ -2266,21 +2279,21 @@ define amdgpu_kernel void @s_fdiv_f32_denorms_correctly_rounded_divide_sqrt(ptr
; GFX6-SLOWFMA-NEXT: v_fma_f32 v3, v4, v2, v3
; GFX6-SLOWFMA-NEXT: v_fma_f32 v0, -v0, v3, v1
; GFX6-SLOWFMA-NEXT: v_div_fmas_f32 v0, v0, v2, v3
-; GFX6-SLOWFMA-NEXT: v_div_fixup_f32 v0, v0, s6, 1.0
-; GFX6-SLOWFMA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-SLOWFMA-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-SLOWFMA-NEXT: v_div_fixup_f32 v0, v0, s2, 1.0
+; GFX6-SLOWFMA-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-SLOWFMA-NEXT: s_endpgm
;
; GFX7-LABEL: s_fdiv_f32_denorms_correctly_rounded_divide_sqrt:
; GFX7: ; %bb.0: ; %entry
-; GFX7-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_div_scale_f32 v0, s[2:3], s6, s6, 1.0
+; GFX7-NEXT: v_div_scale_f32 v0, s[4:5], s2, s2, 1.0
; GFX7-NEXT: v_rcp_f32_e32 v1, v0
-; GFX7-NEXT: v_div_scale_f32 v2, vcc, 1.0, s6, 1.0
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: v_div_scale_f32 v2, vcc, 1.0, s2, 1.0
+; GFX7-NEXT: s_mov_b32 s4, s0
+; GFX7-NEXT: s_mov_b32 s5, s1
; GFX7-NEXT: v_fma_f32 v3, -v0, v1, 1.0
; GFX7-NEXT: v_fma_f32 v1, v3, v1, v1
; GFX7-NEXT: v_mul_f32_e32 v3, v2, v1
@@ -2288,17 +2301,16 @@ define amdgpu_kernel void @s_fdiv_f32_denorms_correctly_rounded_divide_sqrt(ptr
; GFX7-NEXT: v_fma_f32 v3, v4, v1, v3
; GFX7-NEXT: v_fma_f32 v0, -v0, v3, v2
; GFX7-NEXT: v_div_fmas_f32 v0, v0, v1, v3
-; GFX7-NEXT: v_div_fixup_f32 v0, v0, s6, 1.0
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: v_div_fixup_f32 v0, v0, s2, 1.0
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX8-LABEL: s_fdiv_f32_denorms_correctly_rounded_divide_sqrt:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_div_scale_f32 v0, s[0:1], s2, s2, 1.0
+; GFX8-NEXT: v_div_scale_f32 v0, s[4:5], s2, s2, 1.0
; GFX8-NEXT: v_div_scale_f32 v1, vcc, 1.0, s2, 1.0
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8-NEXT: v_rcp_f32_e32 v2, v0
; GFX8-NEXT: v_fma_f32 v3, -v0, v2, 1.0
; GFX8-NEXT: v_fma_f32 v2, v3, v2, v2
@@ -2306,20 +2318,18 @@ define amdgpu_kernel void @s_fdiv_f32_denorms_correctly_rounded_divide_sqrt(ptr
; GFX8-NEXT: v_fma_f32 v4, -v0, v3, v1
; GFX8-NEXT: v_fma_f32 v3, v4, v2, v3
; GFX8-NEXT: v_fma_f32 v0, -v0, v3, v1
-; GFX8-NEXT: v_div_fmas_f32 v0, v0, v2, v3
-; GFX8-NEXT: v_div_fixup_f32 v2, v0, s2, 1.0
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_div_fmas_f32 v2, v0, v2, v3
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_div_fixup_f32 v2, v2, s2, 1.0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX10-LABEL: s_fdiv_f32_denorms_correctly_rounded_divide_sqrt:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_div_scale_f32 v0, s0, s2, s2, 1.0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: v_div_scale_f32 v0, s3, s2, s2, 1.0
; GFX10-NEXT: v_rcp_f32_e32 v1, v0
; GFX10-NEXT: v_fma_f32 v2, -v0, v1, 1.0
; GFX10-NEXT: v_fmac_f32_e32 v1, v2, v1
@@ -2331,15 +2341,12 @@ define amdgpu_kernel void @s_fdiv_f32_denorms_correctly_rounded_divide_sqrt(ptr
; GFX10-NEXT: v_div_fmas_f32 v0, v0, v1, v3
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_div_fixup_f32 v0, v0, s2, 1.0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_fdiv_f32_denorms_correctly_rounded_divide_sqrt:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_div_scale_f32 v0, null, s2, s2, 1.0
; GFX11-NEXT: v_rcp_f32_e32 v1, v0
diff --git a/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll b/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
index da7cae86cf1ea..5e982bde33205 100644
--- a/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
+++ b/llvm/test/CodeGen/AMDGPU/fdiv32-to-rcp-folding.ll
@@ -468,8 +468,7 @@ define amdgpu_kernel void @div_v4_c_by_minus_x_25ulp(ptr addrspace(1) %arg) {
define amdgpu_kernel void @div_v_by_x_25ulp(ptr addrspace(1) %arg, float %num) {
; GCN-DENORM-LABEL: div_v_by_x_25ulp:
; GCN-DENORM: ; %bb.0:
-; GCN-DENORM-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-DENORM-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-DENORM-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-DENORM-NEXT: v_mov_b32_e32 v0, 0
; GCN-DENORM-NEXT: s_waitcnt lgkmcnt(0)
; GCN-DENORM-NEXT: s_load_dword s3, s[0:1], 0x0
@@ -487,8 +486,7 @@ define amdgpu_kernel void @div_v_by_x_25ulp(ptr addrspace(1) %arg, float %num) {
;
; GCN-FLUSH-LABEL: div_v_by_x_25ulp:
; GCN-FLUSH: ; %bb.0:
-; GCN-FLUSH-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-FLUSH-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-FLUSH-NEXT: v_mov_b32_e32 v0, 0x6f800000
; GCN-FLUSH-NEXT: v_mov_b32_e32 v1, 0x2f800000
; GCN-FLUSH-NEXT: v_mov_b32_e32 v2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/flat-address-space.ll b/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
index 20ad0dd45bbfd..eba1266e44094 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-address-space.ll
@@ -7,13 +7,12 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -mattr=-real-true16 -amdgpu-enable-vopd=0 < %s | FileCheck -check-prefixes=GCN,GFX10PLUS,GFX11-FAKE16 %s
; GCN-LABEL: {{^}}store_flat_i32:
-; GCN-DAG: s_load_{{dwordx2|b64}} s[[[LO_SREG:[0-9]+]]:[[HI_SREG:[0-9]+]]],
-; GCN-DAG: s_load_{{dword|b32}} s[[SDATA:[0-9]+]],
+; GCN-DAG: s_load_{{dwordx4|b128}}
; GCN: s_waitcnt lgkmcnt(0)
-; GCN-DAG: v_mov_b32_e32 v[[DATA:[0-9]+]], s[[SDATA]]
-; GCN-DAG: v_mov_b32_e32 v[[LO_VREG:[0-9]+]], s[[LO_SREG]]
-; GCN-DAG: v_mov_b32_e32 v[[HI_VREG:[0-9]+]], s[[HI_SREG]]
-; GCN: flat_store_{{dword|b32}} v[[[LO_VREG]]:[[HI_VREG]]], v[[DATA]]
+; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
+; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
+; GCN-DAG: v_mov_b32_e32 v{{[0-9]+}}, s{{[0-9]+}}
+; GCN: flat_store_{{dword|b32}} v{{\[[0-9]+:[0-9]+\]}}, v{{[0-9]+}}
define amdgpu_kernel void @store_flat_i32(ptr addrspace(1) %gptr, i32 %x) #0 {
%fptr = addrspacecast ptr addrspace(1) %gptr to ptr
store volatile i32 %x, ptr %fptr, align 4
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch-init.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch-init.ll
index 37cbd2d926413..87ea552380830 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch-init.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch-init.ll
@@ -112,14 +112,13 @@ define amdgpu_kernel void @kernel_calls_no_stack() {
define amdgpu_kernel void @test(ptr addrspace(1) %out, i32 %in) {
; GCN-LABEL: test:
; GCN: ; %bb.0:
-; GCN-NEXT: s_clause 0x1
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GCN-NEXT: s_load_dword vcc_lo, s[4:5], 0x8
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GCN-NEXT: ; implicit-def: $vgpr0 : SGPR spill to VGPR lane
-; GCN-NEXT: ; kill: killed $sgpr4_sgpr5
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_writelane_b32 v0, s0, 0
; GCN-NEXT: v_writelane_b32 v0, s1, 1
+; GCN-NEXT: v_writelane_b32 v0, s2, 2
+; GCN-NEXT: v_writelane_b32 v0, s3, 3
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ;;#ASMSTART
@@ -128,6 +127,8 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, i32 %in) {
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_readlane_b32 s0, v0, 0
+; GCN-NEXT: v_readlane_b32 s2, v0, 2
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ;;#ASMSTART
@@ -150,9 +151,8 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, i32 %in) {
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: ;;#ASMEND
-; GCN-NEXT: v_readlane_b32 s0, v0, 0
-; GCN-NEXT: v_mov_b32_e32 v1, vcc_lo
; GCN-NEXT: v_readlane_b32 s1, v0, 1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
; GCN-NEXT: v_mov_b32_e32 v2, 0
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: ;;#ASMEND
@@ -208,6 +208,7 @@ define amdgpu_kernel void @test(ptr addrspace(1) %out, i32 %in) {
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: ;;#ASMEND
+; GCN-NEXT: v_readlane_b32 s3, v0, 3
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ;;#ASMSTART
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
index b35f07002a48a..c2757998274cc 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics.ll
@@ -8,8 +8,7 @@
define amdgpu_kernel void @atomic_add_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_add_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -23,8 +22,7 @@ define amdgpu_kernel void @atomic_add_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_add_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -38,8 +36,7 @@ define amdgpu_kernel void @atomic_add_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_add_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -51,9 +48,7 @@ define amdgpu_kernel void @atomic_add_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_add_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -72,8 +67,7 @@ entry:
define amdgpu_kernel void @atomic_add_i32_max_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_add_i32_max_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 0xffc
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -87,8 +81,7 @@ define amdgpu_kernel void @atomic_add_i32_max_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_add_i32_max_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 0xffc
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -102,8 +95,7 @@ define amdgpu_kernel void @atomic_add_i32_max_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_add_i32_max_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -115,9 +107,7 @@ define amdgpu_kernel void @atomic_add_i32_max_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_add_i32_max_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -136,8 +126,7 @@ entry:
define amdgpu_kernel void @atomic_add_i32_max_offset_p1(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_add_i32_max_offset_p1:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 0x1000
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -151,8 +140,7 @@ define amdgpu_kernel void @atomic_add_i32_max_offset_p1(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_add_i32_max_offset_p1:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 0x1000
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -166,8 +154,7 @@ define amdgpu_kernel void @atomic_add_i32_max_offset_p1(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_add_i32_max_offset_p1:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -181,9 +168,7 @@ define amdgpu_kernel void @atomic_add_i32_max_offset_p1(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_add_i32_max_offset_p1:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v0, s0, 0x1000, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -279,18 +264,17 @@ entry:
define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_add_i32_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_add v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -298,18 +282,17 @@ define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GCN2-LABEL: atomic_add_i32_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_add v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -317,16 +300,15 @@ define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GCN3-LABEL: atomic_add_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_add v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -334,17 +316,16 @@ define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GFX11-LABEL: atomic_add_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_add_u32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -454,8 +435,7 @@ entry:
define amdgpu_kernel void @atomic_add_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_add_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -467,8 +447,7 @@ define amdgpu_kernel void @atomic_add_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_add_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -480,8 +459,7 @@ define amdgpu_kernel void @atomic_add_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_add_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -493,9 +471,7 @@ define amdgpu_kernel void @atomic_add_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_add_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -583,16 +559,15 @@ entry:
define amdgpu_kernel void @atomic_add_i32_addr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_add_i32_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_add v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -600,16 +575,15 @@ define amdgpu_kernel void @atomic_add_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GCN2-LABEL: atomic_add_i32_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_add v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -617,16 +591,15 @@ define amdgpu_kernel void @atomic_add_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GCN3-LABEL: atomic_add_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_add v[0:1], v2
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -634,17 +607,16 @@ define amdgpu_kernel void @atomic_add_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GFX11-LABEL: atomic_add_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_add_u32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -748,8 +720,7 @@ entry:
define amdgpu_kernel void @atomic_and_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_and_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -763,8 +734,7 @@ define amdgpu_kernel void @atomic_and_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_and_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -778,8 +748,7 @@ define amdgpu_kernel void @atomic_and_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_and_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -791,9 +760,7 @@ define amdgpu_kernel void @atomic_and_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_and_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -887,18 +854,17 @@ entry:
define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_and_i32_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_and v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -906,18 +872,17 @@ define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GCN2-LABEL: atomic_and_i32_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_and v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -925,16 +890,15 @@ define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GCN3-LABEL: atomic_and_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_and v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -942,17 +906,16 @@ define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GFX11-LABEL: atomic_and_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_and_b32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1062,8 +1025,7 @@ entry:
define amdgpu_kernel void @atomic_and_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_and_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -1075,8 +1037,7 @@ define amdgpu_kernel void @atomic_and_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_and_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -1088,8 +1049,7 @@ define amdgpu_kernel void @atomic_and_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_and_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -1101,9 +1061,7 @@ define amdgpu_kernel void @atomic_and_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_and_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -1191,16 +1149,15 @@ entry:
define amdgpu_kernel void @atomic_and_i32_addr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_and_i32_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_and v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -1208,16 +1165,15 @@ define amdgpu_kernel void @atomic_and_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GCN2-LABEL: atomic_and_i32_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_and v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -1225,16 +1181,15 @@ define amdgpu_kernel void @atomic_and_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GCN3-LABEL: atomic_and_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_and v[0:1], v2
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -1242,17 +1197,16 @@ define amdgpu_kernel void @atomic_and_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GFX11-LABEL: atomic_and_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_and_b32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1356,8 +1310,7 @@ entry:
define amdgpu_kernel void @atomic_sub_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_sub_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -1371,8 +1324,7 @@ define amdgpu_kernel void @atomic_sub_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_sub_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -1386,8 +1338,7 @@ define amdgpu_kernel void @atomic_sub_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_sub_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -1399,9 +1350,7 @@ define amdgpu_kernel void @atomic_sub_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_sub_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -1495,18 +1444,17 @@ entry:
define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_sub_i32_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_sub v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -1514,18 +1462,17 @@ define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GCN2-LABEL: atomic_sub_i32_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_sub v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -1533,16 +1480,15 @@ define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GCN3-LABEL: atomic_sub_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_sub v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -1550,17 +1496,16 @@ define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GFX11-LABEL: atomic_sub_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_sub_u32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1670,8 +1615,7 @@ entry:
define amdgpu_kernel void @atomic_sub_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_sub_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -1683,8 +1627,7 @@ define amdgpu_kernel void @atomic_sub_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_sub_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -1696,8 +1639,7 @@ define amdgpu_kernel void @atomic_sub_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_sub_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -1709,9 +1651,7 @@ define amdgpu_kernel void @atomic_sub_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_sub_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -1799,16 +1739,15 @@ entry:
define amdgpu_kernel void @atomic_sub_i32_addr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_sub_i32_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_sub v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -1816,16 +1755,15 @@ define amdgpu_kernel void @atomic_sub_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GCN2-LABEL: atomic_sub_i32_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_sub v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -1833,16 +1771,15 @@ define amdgpu_kernel void @atomic_sub_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GCN3-LABEL: atomic_sub_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_sub v[0:1], v2
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -1850,17 +1787,16 @@ define amdgpu_kernel void @atomic_sub_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GFX11-LABEL: atomic_sub_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_sub_u32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -1964,8 +1900,7 @@ entry:
define amdgpu_kernel void @atomic_max_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_max_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -1978,8 +1913,7 @@ define amdgpu_kernel void @atomic_max_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_max_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -1992,8 +1926,7 @@ define amdgpu_kernel void @atomic_max_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_max_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -2004,9 +1937,7 @@ define amdgpu_kernel void @atomic_max_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_max_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -2098,69 +2029,65 @@ entry:
define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i32_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_smax v[0:1], v2
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i32_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_smax v[0:1], v2
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_smax v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_max_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_max_i32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -2268,8 +2195,7 @@ entry:
define amdgpu_kernel void @atomic_max_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_max_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -2280,8 +2206,7 @@ define amdgpu_kernel void @atomic_max_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_max_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -2292,8 +2217,7 @@ define amdgpu_kernel void @atomic_max_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_max_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -2304,9 +2228,7 @@ define amdgpu_kernel void @atomic_max_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_max_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -2392,65 +2314,61 @@ entry:
define amdgpu_kernel void @atomic_max_i32_addr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_max_i32_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_smax v[0:1], v2
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_max_i32_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_smax v[0:1], v2
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_max_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_smax v[0:1], v2
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_max_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_max_i32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -2552,8 +2470,7 @@ entry:
define amdgpu_kernel void @atomic_umax_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_umax_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -2566,8 +2483,7 @@ define amdgpu_kernel void @atomic_umax_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_umax_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -2580,8 +2496,7 @@ define amdgpu_kernel void @atomic_umax_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_umax_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -2592,9 +2507,7 @@ define amdgpu_kernel void @atomic_umax_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_umax_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -2686,69 +2599,65 @@ entry:
define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i32_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_umax v[0:1], v2
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umax_i32_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_umax v[0:1], v2
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umax_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_umax v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_umax_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_max_u32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -2856,8 +2765,7 @@ entry:
define amdgpu_kernel void @atomic_umax_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_umax_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -2868,8 +2776,7 @@ define amdgpu_kernel void @atomic_umax_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_umax_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -2880,8 +2787,7 @@ define amdgpu_kernel void @atomic_umax_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_umax_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -2892,9 +2798,7 @@ define amdgpu_kernel void @atomic_umax_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_umax_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -2980,65 +2884,61 @@ entry:
define amdgpu_kernel void @atomic_umax_i32_addr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_umax_i32_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_umax v[0:1], v2
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umax_i32_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_umax v[0:1], v2
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umax_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_umax v[0:1], v2
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_umax_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_max_u32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -3140,8 +3040,7 @@ entry:
define amdgpu_kernel void @atomic_min_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_min_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -3154,8 +3053,7 @@ define amdgpu_kernel void @atomic_min_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_min_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -3168,8 +3066,7 @@ define amdgpu_kernel void @atomic_min_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_min_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -3180,9 +3077,7 @@ define amdgpu_kernel void @atomic_min_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_min_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -3274,69 +3169,65 @@ entry:
define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i32_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_smin v[0:1], v2
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i32_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_smin v[0:1], v2
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_smin v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_min_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_min_i32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -3444,8 +3335,7 @@ entry:
define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_min_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -3456,8 +3346,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_min_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -3468,8 +3357,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_min_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -3480,9 +3368,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_min_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -3568,65 +3454,61 @@ entry:
define amdgpu_kernel void @atomic_min_i32_addr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_min_i32_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_smin v[0:1], v2
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i32_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_smin v[0:1], v2
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_smin v[0:1], v2
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_min_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_min_i32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -3728,8 +3610,7 @@ entry:
define amdgpu_kernel void @atomic_umin_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_umin_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -3742,8 +3623,7 @@ define amdgpu_kernel void @atomic_umin_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_umin_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -3756,8 +3636,7 @@ define amdgpu_kernel void @atomic_umin_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_umin_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -3768,9 +3647,7 @@ define amdgpu_kernel void @atomic_umin_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_umin_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -3862,69 +3739,65 @@ entry:
define amdgpu_kernel void @atomic_umin_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_umin_i32_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
-; GCN1-NEXT: s_add_u32 s0, s0, 16
-; GCN1-NEXT: s_addc_u32 s1, s1, 0
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
+; GCN1-NEXT: s_add_u32 s0, s0, 16
+; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_umin v[0:1], v2
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umin_i32_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_umin v[0:1], v2
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umin_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_umin v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_umin_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_min_u32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -4032,8 +3905,7 @@ entry:
define amdgpu_kernel void @atomic_umin_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_umin_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -4044,8 +3916,7 @@ define amdgpu_kernel void @atomic_umin_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_umin_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -4056,8 +3927,7 @@ define amdgpu_kernel void @atomic_umin_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_umin_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -4068,9 +3938,7 @@ define amdgpu_kernel void @atomic_umin_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_umin_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -4156,65 +4024,61 @@ entry:
define amdgpu_kernel void @atomic_umin_i32_addr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_umin_i32_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_umin v[0:1], v2
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_umin_i32_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_umin v[0:1], v2
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_umin_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_umin v[0:1], v2
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_umin_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_min_u32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -4316,8 +4180,7 @@ entry:
define amdgpu_kernel void @atomic_or_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_or_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -4331,8 +4194,7 @@ define amdgpu_kernel void @atomic_or_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_or_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -4346,8 +4208,7 @@ define amdgpu_kernel void @atomic_or_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_or_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -4359,9 +4220,7 @@ define amdgpu_kernel void @atomic_or_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_or_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -4455,18 +4314,17 @@ entry:
define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_or_i32_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_or v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -4474,18 +4332,17 @@ define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr %out, i32 %in, i64 %i
;
; GCN2-LABEL: atomic_or_i32_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_or v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -4493,16 +4350,15 @@ define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr %out, i32 %in, i64 %i
;
; GCN3-LABEL: atomic_or_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_or v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -4510,17 +4366,16 @@ define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr %out, i32 %in, i64 %i
;
; GFX11-LABEL: atomic_or_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_or_b32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -4630,8 +4485,7 @@ entry:
define amdgpu_kernel void @atomic_or_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_or_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -4643,8 +4497,7 @@ define amdgpu_kernel void @atomic_or_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_or_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -4656,8 +4509,7 @@ define amdgpu_kernel void @atomic_or_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_or_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -4669,9 +4521,7 @@ define amdgpu_kernel void @atomic_or_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_or_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -4759,16 +4609,15 @@ entry:
define amdgpu_kernel void @atomic_or_i32_addr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_or_i32_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_or v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -4776,16 +4625,15 @@ define amdgpu_kernel void @atomic_or_i32_addr64(ptr %out, i32 %in, i64 %index) {
;
; GCN2-LABEL: atomic_or_i32_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_or v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -4793,16 +4641,15 @@ define amdgpu_kernel void @atomic_or_i32_addr64(ptr %out, i32 %in, i64 %index) {
;
; GCN3-LABEL: atomic_or_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_or v[0:1], v2
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -4810,17 +4657,16 @@ define amdgpu_kernel void @atomic_or_i32_addr64(ptr %out, i32 %in, i64 %index) {
;
; GFX11-LABEL: atomic_or_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_or_b32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -4924,8 +4770,7 @@ entry:
define amdgpu_kernel void @atomic_xchg_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_xchg_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -4939,8 +4784,7 @@ define amdgpu_kernel void @atomic_xchg_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_xchg_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -4954,8 +4798,7 @@ define amdgpu_kernel void @atomic_xchg_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_xchg_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -4967,9 +4810,7 @@ define amdgpu_kernel void @atomic_xchg_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_xchg_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -4988,8 +4829,7 @@ entry:
define amdgpu_kernel void @atomic_xchg_f32_offset(ptr %out, float %in) {
; GCN1-LABEL: atomic_xchg_f32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -5003,8 +4843,7 @@ define amdgpu_kernel void @atomic_xchg_f32_offset(ptr %out, float %in) {
;
; GCN2-LABEL: atomic_xchg_f32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -5018,8 +4857,7 @@ define amdgpu_kernel void @atomic_xchg_f32_offset(ptr %out, float %in) {
;
; GCN3-LABEL: atomic_xchg_f32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -5031,9 +4869,7 @@ define amdgpu_kernel void @atomic_xchg_f32_offset(ptr %out, float %in) {
;
; GFX11-LABEL: atomic_xchg_f32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -5127,18 +4963,17 @@ entry:
define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_xchg_i32_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_swap v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -5146,18 +4981,17 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(ptr %out, i32 %in, i64
;
; GCN2-LABEL: atomic_xchg_i32_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_swap v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -5165,16 +4999,15 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(ptr %out, i32 %in, i64
;
; GCN3-LABEL: atomic_xchg_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_swap v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -5182,17 +5015,16 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(ptr %out, i32 %in, i64
;
; GFX11-LABEL: atomic_xchg_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_swap_b32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5302,8 +5134,7 @@ entry:
define amdgpu_kernel void @atomic_xchg_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_xchg_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -5315,8 +5146,7 @@ define amdgpu_kernel void @atomic_xchg_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_xchg_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -5328,8 +5158,7 @@ define amdgpu_kernel void @atomic_xchg_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_xchg_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -5341,9 +5170,7 @@ define amdgpu_kernel void @atomic_xchg_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_xchg_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -5431,16 +5258,15 @@ entry:
define amdgpu_kernel void @atomic_xchg_i32_addr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_xchg_i32_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_swap v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -5448,16 +5274,15 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GCN2-LABEL: atomic_xchg_i32_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_swap v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -5465,16 +5290,15 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GCN3-LABEL: atomic_xchg_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_swap v[0:1], v2
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -5482,17 +5306,16 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GFX11-LABEL: atomic_xchg_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_swap_b32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -5602,11 +5425,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr %out, i32 %in, i32 %old
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v2, s2
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v3, s3
-; GCN1-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
+; GCN1-NEXT: v_mov_b32_e32 v3, s1
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: s_endpgm
@@ -5617,11 +5440,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr %out, i32 %in, i32 %old
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v2, s2
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v3, s3
-; GCN2-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
+; GCN2-NEXT: v_mov_b32_e32 v3, s1
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: s_endpgm
@@ -5631,8 +5454,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr %out, i32 %in, i32 %old
; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: v_mov_b32_e32 v1, s1
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: v_mov_b32_e32 v3, s3
; GCN3-NEXT: flat_atomic_cmpswap v[0:1], v[2:3] offset:16
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -5665,11 +5488,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_offset(ptr %out, ptr %out2, i3
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
-; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
-; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v3, s5
-; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GCN1-NEXT: v_mov_b32_e32 v3, s1
+; GCN1-NEXT: v_mov_b32_e32 v0, s4
+; GCN1-NEXT: v_mov_b32_e32 v1, s5
+; GCN1-NEXT: v_mov_b32_e32 v2, s0
+; GCN1-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_mov_b32_e32 v0, s2
@@ -5684,11 +5507,11 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_offset(ptr %out, ptr %out2, i3
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
-; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
-; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v3, s5
-; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
+; GCN2-NEXT: v_mov_b32_e32 v3, s1
+; GCN2-NEXT: v_mov_b32_e32 v0, s4
+; GCN2-NEXT: v_mov_b32_e32 v1, s5
+; GCN2-NEXT: v_mov_b32_e32 v2, s0
+; GCN2-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_mov_b32_e32 v0, s2
@@ -5702,8 +5525,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_offset(ptr %out, ptr %out2, i3
; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s1
+; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -5739,19 +5562,16 @@ entry:
define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(ptr %out, i32 %in, i64 %index, i32 %old) {
; GCN1-LABEL: atomic_cmpxchg_i32_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xf
+; GCN1-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: v_mov_b32_e32 v0, s6
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s6
; GCN1-NEXT: v_mov_b32_e32 v2, s0
; GCN1-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -5760,19 +5580,16 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(ptr %out, i32 %in, i
;
; GCN2-LABEL: atomic_cmpxchg_i32_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x3c
+; GCN2-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: v_mov_b32_e32 v0, s6
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s6
; GCN2-NEXT: v_mov_b32_e32 v2, s0
; GCN2-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -5781,17 +5598,14 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(ptr %out, i32 %in, i
;
; GCN3-LABEL: atomic_cmpxchg_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s7, s[4:5], 0x3c
+; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: v_mov_b32_e32 v0, s6
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[0:1], s[12:13], 2
+; GCN3-NEXT: s_add_u32 s0, s8, s0
+; GCN3-NEXT: s_addc_u32 s1, s9, s1
; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v1, s7
+; GCN3-NEXT: v_mov_b32_e32 v0, s10
+; GCN3-NEXT: v_mov_b32_e32 v1, s14
; GCN3-NEXT: v_mov_b32_e32 v2, s0
; GCN3-NEXT: flat_atomic_cmpswap v[2:3], v[0:1] offset:16
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -5800,17 +5614,13 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(ptr %out, i32 %in, i
;
; GFX11-LABEL: atomic_cmpxchg_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x3
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x3c
+; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
-; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s4
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v[2:3], v[0:1] offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -5828,93 +5638,93 @@ entry:
define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64_offset(ptr %out, ptr %out2, i32 %in, i64 %index, i32 %old) {
; GCN1-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf
-; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s9, s[4:5], 0x11
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xf
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; GCN1-NEXT: v_mov_b32_e32 v0, s8
-; GCN1-NEXT: s_add_u32 s0, s0, s4
-; GCN1-NEXT: s_addc_u32 s1, s1, s5
+; GCN1-NEXT: s_load_dword s3, s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN1-NEXT: s_waitcnt lgkmcnt(0)
+; GCN1-NEXT: s_add_u32 s0, s4, s0
+; GCN1-NEXT: s_addc_u32 s1, s5, s1
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v1, s9
+; GCN1-NEXT: v_mov_b32_e32 v0, s3
+; GCN1-NEXT: v_mov_b32_e32 v1, s2
; GCN1-NEXT: v_mov_b32_e32 v2, s0
; GCN1-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_mov_b32_e32 v0, s2
-; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: v_mov_b32_e32 v0, s6
+; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s9, s[4:5], 0x44
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x3c
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; GCN2-NEXT: v_mov_b32_e32 v0, s8
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_load_dword s3, s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_add_u32 s0, s4, s0
+; GCN2-NEXT: s_addc_u32 s1, s5, s1
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v1, s9
+; GCN2-NEXT: v_mov_b32_e32 v0, s3
+; GCN2-NEXT: v_mov_b32_e32 v1, s2
; GCN2-NEXT: v_mov_b32_e32 v2, s0
; GCN2-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_mov_b32_e32 v0, s2
-; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: v_mov_b32_e32 v0, s6
+; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GCN3-NEXT: s_load_dword s8, s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s9, s[4:5], 0x44
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x3c
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; GCN3-NEXT: v_mov_b32_e32 v0, s8
-; GCN3-NEXT: s_add_u32 s0, s0, s4
-; GCN3-NEXT: s_addc_u32 s1, s1, s5
+; GCN3-NEXT: s_load_dword s3, s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN3-NEXT: s_waitcnt lgkmcnt(0)
+; GCN3-NEXT: s_add_u32 s0, s8, s0
+; GCN3-NEXT: s_addc_u32 s1, s9, s1
; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v1, s9
+; GCN3-NEXT: v_mov_b32_e32 v0, s3
+; GCN3-NEXT: v_mov_b32_e32 v1, s2
; GCN3-NEXT: v_mov_b32_e32 v2, s0
; GCN3-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] offset:16 glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_mov_b32_e32 v0, s2
-; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: v_mov_b32_e32 v0, s10
+; GCN3-NEXT: v_mov_b32_e32 v1, s11
; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x3
-; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s9, s[4:5], 0x44
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x3c
+; GFX11-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x34
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v0, s3
+; GFX11-NEXT: s_add_u32 s0, s8, s0
+; GFX11-NEXT: s_addc_u32 s1, s9, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s0, s4
-; GFX11-NEXT: s_addc_u32 s1, s1, s5
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[2:3], v[0:1] offset:16 glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_dual_mov_b32 v0, s10 :: v_dual_mov_b32 v1, s11
; GFX11-NEXT: flat_store_b32 v[0:1], v2
; GFX11-NEXT: s_endpgm
entry:
@@ -5932,8 +5742,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i32(ptr %out, i32 %in, i32 %old) {
; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: v_mov_b32_e32 v1, s1
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: v_mov_b32_e32 v3, s3
; GCN1-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -5945,8 +5755,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i32(ptr %out, i32 %in, i32 %old) {
; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: v_mov_b32_e32 v1, s1
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: v_mov_b32_e32 v3, s3
; GCN2-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -5958,8 +5768,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i32(ptr %out, i32 %in, i32 %old) {
; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: v_mov_b32_e32 v1, s1
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: v_mov_b32_e32 v3, s3
; GCN3-NEXT: flat_atomic_cmpswap v[0:1], v[2:3]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -5990,8 +5800,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret(ptr %out, ptr %out2, i32 %in,
; GCN1-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: v_mov_b32_e32 v1, s1
+; GCN1-NEXT: v_mov_b32_e32 v2, s4
; GCN1-NEXT: v_mov_b32_e32 v3, s5
; GCN1-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -6007,8 +5817,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret(ptr %out, ptr %out2, i32 %in,
; GCN2-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: v_mov_b32_e32 v1, s1
+; GCN2-NEXT: v_mov_b32_e32 v2, s4
; GCN2-NEXT: v_mov_b32_e32 v3, s5
; GCN2-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -6024,8 +5834,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret(ptr %out, ptr %out2, i32 %in,
; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v1, s1
+; GCN3-NEXT: v_mov_b32_e32 v2, s6
; GCN3-NEXT: v_mov_b32_e32 v3, s7
; GCN3-NEXT: flat_atomic_cmpswap v2, v[0:1], v[2:3] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -6060,17 +5870,14 @@ entry:
define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(ptr %out, i32 %in, i64 %index, i32 %old) {
; GCN1-LABEL: atomic_cmpxchg_i32_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xf
+; GCN1-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: v_mov_b32_e32 v0, s6
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v1, s4
+; GCN1-NEXT: v_mov_b32_e32 v0, s2
+; GCN1-NEXT: v_mov_b32_e32 v1, s6
; GCN1-NEXT: v_mov_b32_e32 v2, s0
; GCN1-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -6079,17 +5886,14 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(ptr %out, i32 %in, i64 %ind
;
; GCN2-LABEL: atomic_cmpxchg_i32_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x3c
+; GCN2-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: v_mov_b32_e32 v0, s6
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v1, s4
+; GCN2-NEXT: v_mov_b32_e32 v0, s2
+; GCN2-NEXT: v_mov_b32_e32 v1, s6
; GCN2-NEXT: v_mov_b32_e32 v2, s0
; GCN2-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -6098,17 +5902,14 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(ptr %out, i32 %in, i64 %ind
;
; GCN3-LABEL: atomic_cmpxchg_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s7, s[4:5], 0x3c
+; GCN3-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: v_mov_b32_e32 v0, s6
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[0:1], s[12:13], 2
+; GCN3-NEXT: s_add_u32 s0, s8, s0
+; GCN3-NEXT: s_addc_u32 s1, s9, s1
; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v1, s7
+; GCN3-NEXT: v_mov_b32_e32 v0, s10
+; GCN3-NEXT: v_mov_b32_e32 v1, s14
; GCN3-NEXT: v_mov_b32_e32 v2, s0
; GCN3-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -6117,17 +5918,13 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(ptr %out, i32 %in, i64 %ind
;
; GFX11-LABEL: atomic_cmpxchg_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x3
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x3c
+; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
-; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s4
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v[2:3], v[0:1]
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
@@ -6144,89 +5941,89 @@ entry:
define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64(ptr %out, ptr %out2, i32 %in, i64 %index, i32 %old) {
; GCN1-LABEL: atomic_cmpxchg_i32_ret_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xf
-; GCN1-NEXT: s_load_dword s8, s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s9, s[4:5], 0x11
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xf
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; GCN1-NEXT: v_mov_b32_e32 v0, s8
-; GCN1-NEXT: s_add_u32 s0, s0, s4
-; GCN1-NEXT: s_addc_u32 s1, s1, s5
+; GCN1-NEXT: s_load_dword s3, s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
+; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN1-NEXT: s_waitcnt lgkmcnt(0)
+; GCN1-NEXT: s_add_u32 s0, s4, s0
+; GCN1-NEXT: s_addc_u32 s1, s5, s1
; GCN1-NEXT: v_mov_b32_e32 v3, s1
-; GCN1-NEXT: v_mov_b32_e32 v1, s9
+; GCN1-NEXT: v_mov_b32_e32 v0, s3
+; GCN1-NEXT: v_mov_b32_e32 v1, s2
; GCN1-NEXT: v_mov_b32_e32 v2, s0
; GCN1-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
-; GCN1-NEXT: v_mov_b32_e32 v0, s2
-; GCN1-NEXT: v_mov_b32_e32 v1, s3
+; GCN1-NEXT: v_mov_b32_e32 v0, s6
+; GCN1-NEXT: v_mov_b32_e32 v1, s7
; GCN1-NEXT: flat_store_dword v[0:1], v2
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_cmpxchg_i32_ret_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GCN2-NEXT: s_load_dword s8, s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s9, s[4:5], 0x44
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x3c
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; GCN2-NEXT: v_mov_b32_e32 v0, s8
-; GCN2-NEXT: s_add_u32 s0, s0, s4
-; GCN2-NEXT: s_addc_u32 s1, s1, s5
+; GCN2-NEXT: s_load_dword s3, s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN2-NEXT: s_waitcnt lgkmcnt(0)
+; GCN2-NEXT: s_add_u32 s0, s4, s0
+; GCN2-NEXT: s_addc_u32 s1, s5, s1
; GCN2-NEXT: v_mov_b32_e32 v3, s1
-; GCN2-NEXT: v_mov_b32_e32 v1, s9
+; GCN2-NEXT: v_mov_b32_e32 v0, s3
+; GCN2-NEXT: v_mov_b32_e32 v1, s2
; GCN2-NEXT: v_mov_b32_e32 v2, s0
; GCN2-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
-; GCN2-NEXT: v_mov_b32_e32 v0, s2
-; GCN2-NEXT: v_mov_b32_e32 v1, s3
+; GCN2-NEXT: v_mov_b32_e32 v0, s6
+; GCN2-NEXT: v_mov_b32_e32 v1, s7
; GCN2-NEXT: flat_store_dword v[0:1], v2
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_cmpxchg_i32_ret_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GCN3-NEXT: s_load_dword s8, s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s9, s[4:5], 0x44
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x3c
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; GCN3-NEXT: v_mov_b32_e32 v0, s8
-; GCN3-NEXT: s_add_u32 s0, s0, s4
-; GCN3-NEXT: s_addc_u32 s1, s1, s5
+; GCN3-NEXT: s_load_dword s3, s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN3-NEXT: s_waitcnt lgkmcnt(0)
+; GCN3-NEXT: s_add_u32 s0, s8, s0
+; GCN3-NEXT: s_addc_u32 s1, s9, s1
; GCN3-NEXT: v_mov_b32_e32 v3, s1
-; GCN3-NEXT: v_mov_b32_e32 v1, s9
+; GCN3-NEXT: v_mov_b32_e32 v0, s3
+; GCN3-NEXT: v_mov_b32_e32 v1, s2
; GCN3-NEXT: v_mov_b32_e32 v2, s0
; GCN3-NEXT: flat_atomic_cmpswap v2, v[2:3], v[0:1] glc
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
-; GCN3-NEXT: v_mov_b32_e32 v0, s2
-; GCN3-NEXT: v_mov_b32_e32 v1, s3
+; GCN3-NEXT: v_mov_b32_e32 v0, s10
+; GCN3-NEXT: v_mov_b32_e32 v1, s11
; GCN3-NEXT: flat_store_dword v[0:1], v2
; GCN3-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x3
-; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s9, s[4:5], 0x44
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x3c
+; GFX11-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x34
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v0, s3
+; GFX11-NEXT: s_add_u32 s0, s8, s0
+; GFX11-NEXT: s_addc_u32 s1, s9, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s0, s4
-; GFX11-NEXT: s_addc_u32 s1, s1, s5
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: flat_atomic_cmpswap_b32 v2, v[2:3], v[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_dual_mov_b32 v0, s10 :: v_dual_mov_b32 v1, s11
; GFX11-NEXT: flat_store_b32 v[0:1], v2
; GFX11-NEXT: s_endpgm
entry:
@@ -6240,8 +6037,7 @@ entry:
define amdgpu_kernel void @atomic_xor_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_xor_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -6255,8 +6051,7 @@ define amdgpu_kernel void @atomic_xor_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_xor_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -6270,8 +6065,7 @@ define amdgpu_kernel void @atomic_xor_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_xor_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -6283,9 +6077,7 @@ define amdgpu_kernel void @atomic_xor_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_xor_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -6379,18 +6171,17 @@ entry:
define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_xor_i32_addr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_xor v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -6398,18 +6189,17 @@ define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GCN2-LABEL: atomic_xor_i32_addr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_xor v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -6417,16 +6207,15 @@ define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GCN3-LABEL: atomic_xor_i32_addr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_xor v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -6434,17 +6223,16 @@ define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr %out, i32 %in, i64 %
;
; GFX11-LABEL: atomic_xor_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_xor_b32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -6554,8 +6342,7 @@ entry:
define amdgpu_kernel void @atomic_xor_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_xor_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -6567,8 +6354,7 @@ define amdgpu_kernel void @atomic_xor_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_xor_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -6580,8 +6366,7 @@ define amdgpu_kernel void @atomic_xor_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_xor_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -6593,9 +6378,7 @@ define amdgpu_kernel void @atomic_xor_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_xor_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -6683,16 +6466,15 @@ entry:
define amdgpu_kernel void @atomic_xor_i32_addr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_xor_i32_addr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_xor v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -6700,16 +6482,15 @@ define amdgpu_kernel void @atomic_xor_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GCN2-LABEL: atomic_xor_i32_addr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_xor v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -6717,16 +6498,15 @@ define amdgpu_kernel void @atomic_xor_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GCN3-LABEL: atomic_xor_i32_addr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_xor v[0:1], v2
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -6734,17 +6514,16 @@ define amdgpu_kernel void @atomic_xor_i32_addr64(ptr %out, i32 %in, i64 %index)
;
; GFX11-LABEL: atomic_xor_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_xor_b32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -9069,8 +8848,7 @@ define amdgpu_kernel void @atomic_store_bf16(bfloat %in, ptr %out) {
define amdgpu_kernel void @atomic_inc_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_inc_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -9084,8 +8862,7 @@ define amdgpu_kernel void @atomic_inc_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_inc_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -9099,8 +8876,7 @@ define amdgpu_kernel void @atomic_inc_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_inc_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -9112,9 +8888,7 @@ define amdgpu_kernel void @atomic_inc_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_inc_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -9133,8 +8907,7 @@ entry:
define amdgpu_kernel void @atomic_inc_i32_max_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_inc_i32_max_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 0xffc
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -9148,8 +8921,7 @@ define amdgpu_kernel void @atomic_inc_i32_max_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_inc_i32_max_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 0xffc
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -9163,8 +8935,7 @@ define amdgpu_kernel void @atomic_inc_i32_max_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_inc_i32_max_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -9176,9 +8947,7 @@ define amdgpu_kernel void @atomic_inc_i32_max_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_inc_i32_max_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -9197,8 +8966,7 @@ entry:
define amdgpu_kernel void @atomic_inc_i32_max_offset_p1(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_inc_i32_max_offset_p1:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 0x1000
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -9212,8 +8980,7 @@ define amdgpu_kernel void @atomic_inc_i32_max_offset_p1(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_inc_i32_max_offset_p1:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 0x1000
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -9227,8 +8994,7 @@ define amdgpu_kernel void @atomic_inc_i32_max_offset_p1(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_inc_i32_max_offset_p1:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -9242,9 +9008,7 @@ define amdgpu_kernel void @atomic_inc_i32_max_offset_p1(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_inc_i32_max_offset_p1:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v0, s0, 0x1000, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -9340,18 +9104,17 @@ entry:
define amdgpu_kernel void @atomic_inc_i32_incr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_inc_i32_incr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_inc v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -9359,18 +9122,17 @@ define amdgpu_kernel void @atomic_inc_i32_incr64_offset(ptr %out, i32 %in, i64 %
;
; GCN2-LABEL: atomic_inc_i32_incr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_inc v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -9378,16 +9140,15 @@ define amdgpu_kernel void @atomic_inc_i32_incr64_offset(ptr %out, i32 %in, i64 %
;
; GCN3-LABEL: atomic_inc_i32_incr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_inc v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -9395,17 +9156,16 @@ define amdgpu_kernel void @atomic_inc_i32_incr64_offset(ptr %out, i32 %in, i64 %
;
; GFX11-LABEL: atomic_inc_i32_incr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -9515,8 +9275,7 @@ entry:
define amdgpu_kernel void @atomic_inc_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_inc_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -9528,8 +9287,7 @@ define amdgpu_kernel void @atomic_inc_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_inc_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -9541,8 +9299,7 @@ define amdgpu_kernel void @atomic_inc_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_inc_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -9554,9 +9311,7 @@ define amdgpu_kernel void @atomic_inc_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_inc_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -9644,16 +9399,15 @@ entry:
define amdgpu_kernel void @atomic_inc_i32_incr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_inc_i32_incr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_inc v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -9661,16 +9415,15 @@ define amdgpu_kernel void @atomic_inc_i32_incr64(ptr %out, i32 %in, i64 %index)
;
; GCN2-LABEL: atomic_inc_i32_incr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_inc v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -9678,16 +9431,15 @@ define amdgpu_kernel void @atomic_inc_i32_incr64(ptr %out, i32 %in, i64 %index)
;
; GCN3-LABEL: atomic_inc_i32_incr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_inc v[0:1], v2
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -9695,17 +9447,16 @@ define amdgpu_kernel void @atomic_inc_i32_incr64(ptr %out, i32 %in, i64 %index)
;
; GFX11-LABEL: atomic_inc_i32_incr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_inc_u32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -9809,8 +9560,7 @@ entry:
define amdgpu_kernel void @atomic_dec_i32_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_dec_i32_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -9824,8 +9574,7 @@ define amdgpu_kernel void @atomic_dec_i32_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_dec_i32_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -9839,8 +9588,7 @@ define amdgpu_kernel void @atomic_dec_i32_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_dec_i32_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -9852,9 +9600,7 @@ define amdgpu_kernel void @atomic_dec_i32_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_dec_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -9873,8 +9619,7 @@ entry:
define amdgpu_kernel void @atomic_dec_i32_max_offset(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_dec_i32_max_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 0xffc
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -9888,8 +9633,7 @@ define amdgpu_kernel void @atomic_dec_i32_max_offset(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_dec_i32_max_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 0xffc
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -9903,8 +9647,7 @@ define amdgpu_kernel void @atomic_dec_i32_max_offset(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_dec_i32_max_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -9916,9 +9659,7 @@ define amdgpu_kernel void @atomic_dec_i32_max_offset(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_dec_i32_max_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -9937,8 +9678,7 @@ entry:
define amdgpu_kernel void @atomic_dec_i32_max_offset_p1(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_dec_i32_max_offset_p1:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: s_add_u32 s0, s0, 0x1000
; GCN1-NEXT: s_addc_u32 s1, s1, 0
@@ -9952,8 +9692,7 @@ define amdgpu_kernel void @atomic_dec_i32_max_offset_p1(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_dec_i32_max_offset_p1:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: s_add_u32 s0, s0, 0x1000
; GCN2-NEXT: s_addc_u32 s1, s1, 0
@@ -9967,8 +9706,7 @@ define amdgpu_kernel void @atomic_dec_i32_max_offset_p1(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_dec_i32_max_offset_p1:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -9982,9 +9720,7 @@ define amdgpu_kernel void @atomic_dec_i32_max_offset_p1(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_dec_i32_max_offset_p1:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_add_co_u32 v0, s0, 0x1000, s0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -10080,18 +9816,17 @@ entry:
define amdgpu_kernel void @atomic_dec_i32_decr64_offset(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_dec_i32_decr64_offset:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: s_add_u32 s0, s0, 16
; GCN1-NEXT: s_addc_u32 s1, s1, 0
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_dec v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -10099,18 +9834,17 @@ define amdgpu_kernel void @atomic_dec_i32_decr64_offset(ptr %out, i32 %in, i64 %
;
; GCN2-LABEL: atomic_dec_i32_decr64_offset:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: s_add_u32 s0, s0, 16
; GCN2-NEXT: s_addc_u32 s1, s1, 0
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_dec v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -10118,16 +9852,15 @@ define amdgpu_kernel void @atomic_dec_i32_decr64_offset(ptr %out, i32 %in, i64 %
;
; GCN3-LABEL: atomic_dec_i32_decr64_offset:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_dec v[0:1], v2 offset:16
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -10135,17 +9868,16 @@ define amdgpu_kernel void @atomic_dec_i32_decr64_offset(ptr %out, i32 %in, i64 %
;
; GFX11-LABEL: atomic_dec_i32_decr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2 offset:16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -10255,8 +9987,7 @@ entry:
define amdgpu_kernel void @atomic_dec_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_dec_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
@@ -10268,8 +9999,7 @@ define amdgpu_kernel void @atomic_dec_i32(ptr %out, i32 %in) {
;
; GCN2-LABEL: atomic_dec_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
@@ -10281,8 +10011,7 @@ define amdgpu_kernel void @atomic_dec_i32(ptr %out, i32 %in) {
;
; GCN3-LABEL: atomic_dec_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
@@ -10294,9 +10023,7 @@ define amdgpu_kernel void @atomic_dec_i32(ptr %out, i32 %in) {
;
; GFX11-LABEL: atomic_dec_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_mov_b32_e32 v2, s2
@@ -10384,16 +10111,15 @@ entry:
define amdgpu_kernel void @atomic_dec_i32_decr64(ptr %out, i32 %in, i64 %index) {
; GCN1-LABEL: atomic_dec_i32_decr64:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; GCN1-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s4, s[4:5], 0xb
+; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN1-NEXT: s_add_u32 s0, s2, s0
-; GCN1-NEXT: s_addc_u32 s1, s3, s1
+; GCN1-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN1-NEXT: s_add_u32 s0, s0, s4
+; GCN1-NEXT: s_addc_u32 s1, s1, s5
; GCN1-NEXT: v_mov_b32_e32 v0, s0
; GCN1-NEXT: v_mov_b32_e32 v1, s1
-; GCN1-NEXT: v_mov_b32_e32 v2, s4
+; GCN1-NEXT: v_mov_b32_e32 v2, s2
; GCN1-NEXT: flat_atomic_dec v[0:1], v2
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
@@ -10401,16 +10127,15 @@ define amdgpu_kernel void @atomic_dec_i32_decr64(ptr %out, i32 %in, i64 %index)
;
; GCN2-LABEL: atomic_dec_i32_decr64:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN2-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s4, s[4:5], 0x2c
+; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN2-NEXT: s_add_u32 s0, s2, s0
-; GCN2-NEXT: s_addc_u32 s1, s3, s1
+; GCN2-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN2-NEXT: s_add_u32 s0, s0, s4
+; GCN2-NEXT: s_addc_u32 s1, s1, s5
; GCN2-NEXT: v_mov_b32_e32 v0, s0
; GCN2-NEXT: v_mov_b32_e32 v1, s1
-; GCN2-NEXT: v_mov_b32_e32 v2, s4
+; GCN2-NEXT: v_mov_b32_e32 v2, s2
; GCN2-NEXT: flat_atomic_dec v[0:1], v2
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
@@ -10418,16 +10143,15 @@ define amdgpu_kernel void @atomic_dec_i32_decr64(ptr %out, i32 %in, i64 %index)
;
; GCN3-LABEL: atomic_dec_i32_decr64:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GCN3-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GCN3-NEXT: s_add_u32 s0, s2, s0
-; GCN3-NEXT: s_addc_u32 s1, s3, s1
+; GCN3-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GCN3-NEXT: s_add_u32 s0, s0, s4
+; GCN3-NEXT: s_addc_u32 s1, s1, s5
; GCN3-NEXT: v_mov_b32_e32 v0, s0
; GCN3-NEXT: v_mov_b32_e32 v1, s1
-; GCN3-NEXT: v_mov_b32_e32 v2, s6
+; GCN3-NEXT: v_mov_b32_e32 v2, s2
; GCN3-NEXT: flat_atomic_dec v[0:1], v2
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
@@ -10435,17 +10159,16 @@ define amdgpu_kernel void @atomic_dec_i32_decr64(ptr %out, i32 %in, i64 %index)
;
; GFX11-LABEL: atomic_dec_i32_decr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x2c
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX11-NEXT: v_mov_b32_e32 v2, s4
; GFX11-NEXT: flat_atomic_dec_u32 v[0:1], v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
index 278964334b711..493faa5d55930 100644
--- a/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat_atomics_i32_system.ll
@@ -7977,12 +7977,11 @@ entry:
define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
; GCN1-LABEL: atomic_min_i32:
; GCN1: ; %bb.0: ; %entry
-; GCN1-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x9
-; GCN1-NEXT: s_load_dword s2, s[4:5], 0xb
-; GCN1-NEXT: s_mov_b64 s[0:1], 0
+; GCN1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN1-NEXT: s_mov_b64 s[4:5], 0
; GCN1-NEXT: s_waitcnt lgkmcnt(0)
-; GCN1-NEXT: v_mov_b32_e32 v0, s6
-; GCN1-NEXT: v_mov_b32_e32 v1, s7
+; GCN1-NEXT: v_mov_b32_e32 v0, s0
+; GCN1-NEXT: v_mov_b32_e32 v1, s1
; GCN1-NEXT: flat_load_dword v3, v[0:1]
; GCN1-NEXT: .LBB127_1: ; %atomicrmw.start
; GCN1-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -7992,21 +7991,20 @@ define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
; GCN1-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN1-NEXT: buffer_wbinvl1_vol
; GCN1-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GCN1-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN1-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN1-NEXT: v_mov_b32_e32 v3, v2
-; GCN1-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GCN1-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN1-NEXT: s_cbranch_execnz .LBB127_1
; GCN1-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN1-NEXT: s_endpgm
;
; GCN2-LABEL: atomic_min_i32:
; GCN2: ; %bb.0: ; %entry
-; GCN2-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GCN2-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN2-NEXT: s_mov_b64 s[0:1], 0
+; GCN2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN2-NEXT: s_mov_b64 s[4:5], 0
; GCN2-NEXT: s_waitcnt lgkmcnt(0)
-; GCN2-NEXT: v_mov_b32_e32 v0, s6
-; GCN2-NEXT: v_mov_b32_e32 v1, s7
+; GCN2-NEXT: v_mov_b32_e32 v0, s0
+; GCN2-NEXT: v_mov_b32_e32 v1, s1
; GCN2-NEXT: flat_load_dword v3, v[0:1]
; GCN2-NEXT: .LBB127_1: ; %atomicrmw.start
; GCN2-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8016,21 +8014,20 @@ define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
; GCN2-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN2-NEXT: buffer_wbinvl1_vol
; GCN2-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GCN2-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN2-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN2-NEXT: v_mov_b32_e32 v3, v2
-; GCN2-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GCN2-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN2-NEXT: s_cbranch_execnz .LBB127_1
; GCN2-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN2-NEXT: s_endpgm
;
; GCN3-LABEL: atomic_min_i32:
; GCN3: ; %bb.0: ; %entry
-; GCN3-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GCN3-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN3-NEXT: s_mov_b64 s[0:1], 0
+; GCN3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN3-NEXT: s_mov_b64 s[4:5], 0
; GCN3-NEXT: s_waitcnt lgkmcnt(0)
-; GCN3-NEXT: v_mov_b32_e32 v0, s6
-; GCN3-NEXT: v_mov_b32_e32 v1, s7
+; GCN3-NEXT: v_mov_b32_e32 v0, s0
+; GCN3-NEXT: v_mov_b32_e32 v1, s1
; GCN3-NEXT: flat_load_dword v3, v[0:1]
; GCN3-NEXT: .LBB127_1: ; %atomicrmw.start
; GCN3-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -8040,9 +8037,9 @@ define amdgpu_kernel void @atomic_min_i32(ptr %out, i32 %in) {
; GCN3-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GCN3-NEXT: buffer_wbinvl1_vol
; GCN3-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; GCN3-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; GCN3-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GCN3-NEXT: v_mov_b32_e32 v3, v2
-; GCN3-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; GCN3-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GCN3-NEXT: s_cbranch_execnz .LBB127_1
; GCN3-NEXT: ; %bb.2: ; %atomicrmw.end
; GCN3-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fma.f16.gfx11plus.ll b/llvm/test/CodeGen/AMDGPU/fma.f16.gfx11plus.ll
index 5f86f2e48137d..e8b1c9887315e 100644
--- a/llvm/test/CodeGen/AMDGPU/fma.f16.gfx11plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/fma.f16.gfx11plus.ll
@@ -11,20 +11,20 @@ define amdgpu_kernel void @fma_v2f16_divergent(
; GFX11-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; GFX11-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
; GFX11-NEXT: [[S_LOAD_DWORDX8_IMM:%[0-9]+]]:sgpr_256 = S_LOAD_DWORDX8_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s256) from %ir.r.kernarg.offset, align 4, addrspace 4)
- ; GFX11-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 68, 0 :: (dereferenceable invariant load (s32) from %ir.d.kernarg.offset, addrspace 4)
- ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub1
- ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub0
- ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1
+ ; GFX11-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 68, 0 :: (dereferenceable invariant load (s32) from %ir.r.kernarg.offset + 32, addrspace 4)
+ ; GFX11-NEXT: [[COPY2:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub7
+ ; GFX11-NEXT: [[COPY3:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub6
+ ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub5
+ ; GFX11-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub4
+ ; GFX11-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub3
+ ; GFX11-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub2
+ ; GFX11-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub1
+ ; GFX11-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub0
+ ; GFX11-NEXT: [[REG_SEQUENCE:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1
; GFX11-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; GFX11-NEXT: [[COPY4:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub3
- ; GFX11-NEXT: [[COPY5:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub2
- ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1
- ; GFX11-NEXT: [[COPY6:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub5
- ; GFX11-NEXT: [[COPY7:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub4
- ; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1
- ; GFX11-NEXT: [[COPY8:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub7
- ; GFX11-NEXT: [[COPY9:%[0-9]+]]:sreg_32 = COPY [[S_LOAD_DWORDX8_IMM]].sub6
- ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY9]], %subreg.sub0, killed [[COPY8]], %subreg.sub1
+ ; GFX11-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:sreg_64_xexec_xnull = REG_SEQUENCE killed [[COPY7]], %subreg.sub0, killed [[COPY6]], %subreg.sub1
+ ; GFX11-NEXT: [[REG_SEQUENCE2:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY5]], %subreg.sub0, killed [[COPY4]], %subreg.sub1
+ ; GFX11-NEXT: [[REG_SEQUENCE3:%[0-9]+]]:sreg_64 = REG_SEQUENCE killed [[COPY3]], %subreg.sub0, killed [[COPY2]], %subreg.sub1
; GFX11-NEXT: [[S_MOV_B32_:%[0-9]+]]:sreg_32 = S_MOV_B32 1023
; GFX11-NEXT: [[V_AND_B32_e64_:%[0-9]+]]:vgpr_32 = V_AND_B32_e64 [[COPY1]](s32), killed [[S_MOV_B32_]], implicit $exec
; GFX11-NEXT: [[S_MOV_B32_1:%[0-9]+]]:sreg_32 = S_MOV_B32 1
diff --git a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
index 48b334233a075..14f2fe0b6b1dc 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaxnum.ll
@@ -10,14 +10,15 @@ define amdgpu_kernel void @test_fmax_f32_ieee_mode_on(ptr addrspace(1) %out, flo
; GFX8-SDAG-LABEL: test_fmax_f32_ieee_mode_on:
; GFX8-SDAG: ; %bb.0:
; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s5
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v1, 1.0, s4
-; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s3
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v1, 1.0, s2
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
; GFX8-SDAG-NEXT: v_max_f32_e32 v0, v1, v0
-; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-SDAG-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: test_fmax_f32_ieee_mode_on:
@@ -115,109 +116,113 @@ define amdgpu_ps float @test_fmax_f32_ieee_mode_off(float %a, float %b) #0 {
define amdgpu_kernel void @test_fmax_v2f32(ptr addrspace(1) %out, <2 x float> %a, <2 x float> %b) #0 {
; GFX8-SDAG-LABEL: test_fmax_v2f32:
; GFX8-SDAG: ; %bb.0:
-; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s3
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v1, 1.0, s1
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v1, 1.0, s3
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s9
; GFX8-SDAG-NEXT: v_max_f32_e32 v1, v1, v0
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s2
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v2, 1.0, s0
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s8
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v2, 1.0, s2
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
; GFX8-SDAG-NEXT: v_max_f32_e32 v0, v2, v0
; GFX8-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX8-SDAG-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: test_fmax_v2f32:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
-; GFX8-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX8-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_mul_f32_e64 v0, 1.0, s0
-; GFX8-GISEL-NEXT: v_mul_f32_e64 v1, 1.0, s2
-; GFX8-GISEL-NEXT: v_mul_f32_e64 v2, 1.0, s1
-; GFX8-GISEL-NEXT: v_mul_f32_e64 v3, 1.0, s3
+; GFX8-GISEL-NEXT: v_mul_f32_e64 v0, 1.0, s2
+; GFX8-GISEL-NEXT: v_mul_f32_e64 v1, 1.0, s4
+; GFX8-GISEL-NEXT: v_mul_f32_e64 v2, 1.0, s3
+; GFX8-GISEL-NEXT: v_mul_f32_e64 v3, 1.0, s5
; GFX8-GISEL-NEXT: v_max_f32_e32 v0, v0, v1
; GFX8-GISEL-NEXT: v_max_f32_e32 v1, v2, v3
-; GFX8-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX8-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX8-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX8-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: test_fmax_v2f32:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
-; GFX9-SDAG-NEXT: s_mov_b32 s11, 0xf000
-; GFX9-SDAG-NEXT: s_mov_b32 s10, -1
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_max_f32_e64 v0, s3, s3
-; GFX9-SDAG-NEXT: v_max_f32_e64 v1, s1, s1
+; GFX9-SDAG-NEXT: s_mov_b32 s5, s3
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s2
+; GFX9-SDAG-NEXT: v_max_f32_e64 v0, s7, s7
+; GFX9-SDAG-NEXT: v_max_f32_e64 v1, s5, s5
; GFX9-SDAG-NEXT: v_max_f32_e32 v1, v1, v0
-; GFX9-SDAG-NEXT: v_max_f32_e64 v0, s2, s2
-; GFX9-SDAG-NEXT: v_max_f32_e64 v2, s0, s0
+; GFX9-SDAG-NEXT: v_max_f32_e64 v0, s6, s6
+; GFX9-SDAG-NEXT: v_max_f32_e64 v2, s4, s4
+; GFX9-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-SDAG-NEXT: s_mov_b32 s2, -1
; GFX9-SDAG-NEXT: v_max_f32_e32 v0, v2, v0
-; GFX9-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GFX9-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX9-GISEL-LABEL: test_fmax_v2f32:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s10, -1
-; GFX9-GISEL-NEXT: s_mov_b32 s11, 0xf000
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_max_f32_e64 v0, s0, s0
-; GFX9-GISEL-NEXT: v_max_f32_e64 v1, s2, s2
-; GFX9-GISEL-NEXT: v_max_f32_e64 v2, s1, s1
-; GFX9-GISEL-NEXT: v_max_f32_e64 v3, s3, s3
+; GFX9-GISEL-NEXT: v_max_f32_e64 v0, s2, s2
+; GFX9-GISEL-NEXT: v_max_f32_e64 v1, s6, s6
+; GFX9-GISEL-NEXT: v_max_f32_e64 v2, s3, s3
+; GFX9-GISEL-NEXT: v_max_f32_e64 v3, s7, s7
; GFX9-GISEL-NEXT: v_max_f32_e32 v0, v0, v1
; GFX9-GISEL-NEXT: v_max_f32_e32 v1, v2, v3
-; GFX9-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX12-SDAG-LABEL: test_fmax_v2f32:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x1
-; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX12-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s3, s3
-; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s1, s1
-; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s2, s2
-; GFX12-SDAG-NEXT: v_max_num_f32_e64 v3, s0, s0
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s7, s7
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s3, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s6, s6
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v3, s2, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_dual_max_num_f32 v1, v1, v0 :: v_dual_max_num_f32 v0, v3, v2
-; GFX12-SDAG-NEXT: buffer_store_b64 v[0:1], off, s[4:7], null
+; GFX12-SDAG-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: test_fmax_v2f32:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_clause 0x1
-; GFX12-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX12-GISEL-NEXT: s_mov_b32 s7, 0x31016000
+; GFX12-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_max_num_f32_e64 v0, s0, s0
-; GFX12-GISEL-NEXT: v_max_num_f32_e64 v1, s2, s2
-; GFX12-GISEL-NEXT: v_max_num_f32_e64 v2, s1, s1
-; GFX12-GISEL-NEXT: v_max_num_f32_e64 v3, s3, s3
+; GFX12-GISEL-NEXT: v_max_num_f32_e64 v0, s2, s2
+; GFX12-GISEL-NEXT: v_max_num_f32_e64 v1, s4, s4
+; GFX12-GISEL-NEXT: v_max_num_f32_e64 v2, s3, s3
+; GFX12-GISEL-NEXT: v_max_num_f32_e64 v3, s5, s5
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-GISEL-NEXT: s_max_num_f32 s0, s0, s1
-; GFX12-GISEL-NEXT: s_max_num_f32 s1, s2, s3
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s5, v3
+; GFX12-GISEL-NEXT: s_max_num_f32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_max_num_f32 s3, s4, s5
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-GISEL-NEXT: buffer_store_b64 v[0:1], off, s[4:7], null
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX12-GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-GISEL-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX12-GISEL-NEXT: s_endpgm
%val = call <2 x float> @llvm.maxnum.v2f32(<2 x float> %a, <2 x float> %b)
store <2 x float> %val, ptr addrspace(1) %out, align 8
@@ -1861,16 +1866,17 @@ define <3 x float> @test_func_fmax_v3f32(<3 x float> %a, <3 x float> %b) #0 {
define amdgpu_kernel void @test_fmax_f16_v_ieee_on(ptr addrspace(1) %out, half %a, half %b) #0 {
; GFX8-SDAG-LABEL: test_fmax_f16_v_ieee_on:
; GFX8-SDAG: ; %bb.0:
-; GFX8-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT: s_lshr_b32 s4, s6, 16
-; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s0, s0
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s2, s2
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
; GFX8-SDAG-NEXT: v_max_f16_e32 v0, v1, v0
-; GFX8-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: buffer_store_short v0, off, s[4:7], 0
; GFX8-SDAG-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: test_fmax_f16_v_ieee_on:
@@ -1889,16 +1895,17 @@ define amdgpu_kernel void @test_fmax_f16_v_ieee_on(ptr addrspace(1) %out, half %
;
; GFX9-SDAG-LABEL: test_fmax_f16_v_ieee_on:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-SDAG-NEXT: s_mov_b32 s6, -1
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_lshr_b32 s4, s6, 16
-; GFX9-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX9-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX9-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; GFX9-SDAG-NEXT: v_max_f16_e64 v0, s0, s0
+; GFX9-SDAG-NEXT: v_max_f16_e64 v1, s2, s2
+; GFX9-SDAG-NEXT: s_mov_b32 s5, s1
; GFX9-SDAG-NEXT: v_max_f16_e32 v0, v1, v0
-; GFX9-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-SDAG-NEXT: buffer_store_short v0, off, s[4:7], 0
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX9-GISEL-LABEL: test_fmax_f16_v_ieee_on:
diff --git a/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll b/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll
index b478e9a0830eb..a12088621aa06 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin_legacy.ll
@@ -75,9 +75,8 @@ define amdgpu_kernel void @s_test_fmin_legacy_ule_f32_fast(ptr addrspace(1) %out
; GCN-LABEL: {{^}}s_test_fmin_legacy_ule_f32_nnan_src:
; GCN: s_load_dwordx4 s[[[#LOAD:]]:{{[0-9]+}}], s{{\[[0-9]+:[0-9]+\]}}, {{0x9|0x24}}
-; SI: s_mov_b64 s[[[#COPY:]]:{{[0-9]+}}], s{{\[}}[[#LOAD + 2]]:[[#LOAD + 3]]{{\]}}
-; SI-DAG: v_add_f32_e64 [[ADD_A:v[0-9]+]], s[[#COPY]], 1.0
-; SI-DAG: v_add_f32_e64 [[ADD_B:v[0-9]+]], s[[#COPY + 1]], 2.0
+; SI-DAG: v_add_f32_e64 [[ADD_A:v[0-9]+]], s[[#LOAD + 2]], 1.0
+; SI-DAG: v_add_f32_e64 [[ADD_B:v[0-9]+]], s[[#LOAD + 3]], 2.0
; VI-DAG: v_add_f32_e64 [[ADD_A:v[0-9]+]], s[[#LOAD + 2]], 1.0
; VI-DAG: v_add_f32_e64 [[ADD_B:v[0-9]+]], s[[#LOAD + 3]], 2.0
@@ -99,9 +98,8 @@ define amdgpu_kernel void @s_test_fmin_legacy_ule_f32_nnan_src(ptr addrspace(1)
; GCN-LABEL: {{^}}s_test_fmin_legacy_ule_f32_nnan_src_fast:
; GCN: s_load_dwordx4 s[[[#LOAD:]]:{{[0-9]+}}], s{{\[[0-9]+:[0-9]+\]}}, {{0x9|0x24}}
-; SI: s_mov_b64 s[[[#COPY:]]:{{[0-9]+}}], s{{\[}}[[#LOAD + 2]]:[[#LOAD + 3]]{{\]}}
-; SI-DAG: v_add_f32_e64 [[ADD_A:v[0-9]+]], s[[#COPY]], 1.0
-; SI-DAG: v_add_f32_e64 [[ADD_B:v[0-9]+]], s[[#COPY + 1]], 2.0
+; SI-DAG: v_add_f32_e64 [[ADD_A:v[0-9]+]], s[[#LOAD + 2]], 1.0
+; SI-DAG: v_add_f32_e64 [[ADD_B:v[0-9]+]], s[[#LOAD + 3]], 2.0
; VI-DAG: v_add_f32_e64 [[ADD_A:v[0-9]+]], s[[#LOAD + 2]], 1.0
; VI-DAG: v_add_f32_e64 [[ADD_B:v[0-9]+]], s[[#LOAD + 3]], 2.0
diff --git a/llvm/test/CodeGen/AMDGPU/fminnum.ll b/llvm/test/CodeGen/AMDGPU/fminnum.ll
index 2444f7dee860b..8a6c6ca92bf8a 100644
--- a/llvm/test/CodeGen/AMDGPU/fminnum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminnum.ll
@@ -10,14 +10,15 @@ define amdgpu_kernel void @test_fmin_f32_ieee_mode_on(ptr addrspace(1) %out, flo
; GFX8-SDAG-LABEL: test_fmin_f32_ieee_mode_on:
; GFX8-SDAG: ; %bb.0:
; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s5
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v1, 1.0, s4
-; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s3
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v1, 1.0, s2
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
; GFX8-SDAG-NEXT: v_min_f32_e32 v0, v1, v0
-; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-SDAG-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: test_fmin_f32_ieee_mode_on:
@@ -160,109 +161,113 @@ define amdgpu_ps float @test_fmin_f32_ieee_mode_off(float %a, float %b) #0 {
define amdgpu_kernel void @test_fmin_v2f32(ptr addrspace(1) %out, <2 x float> %a, <2 x float> %b) #0 {
; GFX8-SDAG-LABEL: test_fmin_v2f32:
; GFX8-SDAG: ; %bb.0:
-; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-SDAG-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s3
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v1, 1.0, s1
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v1, 1.0, s3
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s9
; GFX8-SDAG-NEXT: v_min_f32_e32 v1, v1, v0
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s2
-; GFX8-SDAG-NEXT: v_mul_f32_e64 v2, 1.0, s0
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v0, 1.0, s8
+; GFX8-SDAG-NEXT: v_mul_f32_e64 v2, 1.0, s2
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
; GFX8-SDAG-NEXT: v_min_f32_e32 v0, v2, v0
; GFX8-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX8-SDAG-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: test_fmin_v2f32:
; GFX8-GISEL: ; %bb.0:
-; GFX8-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
-; GFX8-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX8-GISEL-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-GISEL-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; GFX8-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-GISEL-NEXT: v_mul_f32_e64 v0, 1.0, s0
-; GFX8-GISEL-NEXT: v_mul_f32_e64 v1, 1.0, s2
-; GFX8-GISEL-NEXT: v_mul_f32_e64 v2, 1.0, s1
-; GFX8-GISEL-NEXT: v_mul_f32_e64 v3, 1.0, s3
+; GFX8-GISEL-NEXT: v_mul_f32_e64 v0, 1.0, s2
+; GFX8-GISEL-NEXT: v_mul_f32_e64 v1, 1.0, s4
+; GFX8-GISEL-NEXT: v_mul_f32_e64 v2, 1.0, s3
+; GFX8-GISEL-NEXT: v_mul_f32_e64 v3, 1.0, s5
; GFX8-GISEL-NEXT: v_min_f32_e32 v0, v0, v1
; GFX8-GISEL-NEXT: v_min_f32_e32 v1, v2, v3
-; GFX8-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; GFX8-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX8-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX8-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: test_fmin_v2f32:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
-; GFX9-SDAG-NEXT: s_mov_b32 s11, 0xf000
-; GFX9-SDAG-NEXT: s_mov_b32 s10, -1
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: v_max_f32_e64 v0, s3, s3
-; GFX9-SDAG-NEXT: v_max_f32_e64 v1, s1, s1
+; GFX9-SDAG-NEXT: s_mov_b32 s5, s3
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s2
+; GFX9-SDAG-NEXT: v_max_f32_e64 v0, s7, s7
+; GFX9-SDAG-NEXT: v_max_f32_e64 v1, s5, s5
; GFX9-SDAG-NEXT: v_min_f32_e32 v1, v1, v0
-; GFX9-SDAG-NEXT: v_max_f32_e64 v0, s2, s2
-; GFX9-SDAG-NEXT: v_max_f32_e64 v2, s0, s0
+; GFX9-SDAG-NEXT: v_max_f32_e64 v0, s6, s6
+; GFX9-SDAG-NEXT: v_max_f32_e64 v2, s4, s4
+; GFX9-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-SDAG-NEXT: s_mov_b32 s2, -1
; GFX9-SDAG-NEXT: v_min_f32_e32 v0, v2, v0
-; GFX9-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GFX9-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX9-GISEL-LABEL: test_fmin_v2f32:
; GFX9-GISEL: ; %bb.0:
-; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-GISEL-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
-; GFX9-GISEL-NEXT: s_mov_b32 s10, -1
-; GFX9-GISEL-NEXT: s_mov_b32 s11, 0xf000
+; GFX9-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-GISEL-NEXT: v_max_f32_e64 v0, s0, s0
-; GFX9-GISEL-NEXT: v_max_f32_e64 v1, s2, s2
-; GFX9-GISEL-NEXT: v_max_f32_e64 v2, s1, s1
-; GFX9-GISEL-NEXT: v_max_f32_e64 v3, s3, s3
+; GFX9-GISEL-NEXT: v_max_f32_e64 v0, s2, s2
+; GFX9-GISEL-NEXT: v_max_f32_e64 v1, s6, s6
+; GFX9-GISEL-NEXT: v_max_f32_e64 v2, s3, s3
+; GFX9-GISEL-NEXT: v_max_f32_e64 v3, s7, s7
; GFX9-GISEL-NEXT: v_min_f32_e32 v0, v0, v1
; GFX9-GISEL-NEXT: v_min_f32_e32 v1, v2, v3
-; GFX9-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GFX9-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX9-GISEL-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX9-GISEL-NEXT: s_endpgm
;
; GFX12-SDAG-LABEL: test_fmin_v2f32:
; GFX12-SDAG: ; %bb.0:
; GFX12-SDAG-NEXT: s_clause 0x1
-; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-SDAG-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-SDAG-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-SDAG-NEXT: s_mov_b32 s6, -1
+; GFX12-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX12-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s3, s3
-; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s1, s1
-; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s2, s2
-; GFX12-SDAG-NEXT: v_max_num_f32_e64 v3, s0, s0
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v0, s7, s7
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v1, s3, s3
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v2, s6, s6
+; GFX12-SDAG-NEXT: v_max_num_f32_e64 v3, s2, s2
+; GFX12-SDAG-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-SDAG-NEXT: s_mov_b32 s2, -1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_dual_min_num_f32 v1, v1, v0 :: v_dual_min_num_f32 v0, v3, v2
-; GFX12-SDAG-NEXT: buffer_store_b64 v[0:1], off, s[4:7], null
+; GFX12-SDAG-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX12-SDAG-NEXT: s_endpgm
;
; GFX12-GISEL-LABEL: test_fmin_v2f32:
; GFX12-GISEL: ; %bb.0:
; GFX12-GISEL-NEXT: s_clause 0x1
-; GFX12-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
-; GFX12-GISEL-NEXT: s_mov_b32 s6, -1
-; GFX12-GISEL-NEXT: s_mov_b32 s7, 0x31016000
+; GFX12-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-GISEL-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12-GISEL-NEXT: v_max_num_f32_e64 v0, s0, s0
-; GFX12-GISEL-NEXT: v_max_num_f32_e64 v1, s2, s2
-; GFX12-GISEL-NEXT: v_max_num_f32_e64 v2, s1, s1
-; GFX12-GISEL-NEXT: v_max_num_f32_e64 v3, s3, s3
+; GFX12-GISEL-NEXT: v_max_num_f32_e64 v0, s2, s2
+; GFX12-GISEL-NEXT: v_max_num_f32_e64 v1, s4, s4
+; GFX12-GISEL-NEXT: v_max_num_f32_e64 v2, s3, s3
+; GFX12-GISEL-NEXT: v_max_num_f32_e64 v3, s5, s5
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v1
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s2, v2
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s3, v3
-; GFX12-GISEL-NEXT: s_min_num_f32 s0, s0, s1
-; GFX12-GISEL-NEXT: s_min_num_f32 s1, s2, s3
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s5, v3
+; GFX12-GISEL-NEXT: s_min_num_f32 s2, s2, s3
+; GFX12-GISEL-NEXT: s_min_num_f32 s3, s4, s5
; GFX12-GISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_2)
-; GFX12-GISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX12-GISEL-NEXT: buffer_store_b64 v[0:1], off, s[4:7], null
+; GFX12-GISEL-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX12-GISEL-NEXT: s_mov_b32 s2, -1
+; GFX12-GISEL-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-GISEL-NEXT: buffer_store_b64 v[0:1], off, s[0:3], null
; GFX12-GISEL-NEXT: s_endpgm
%val = call <2 x float> @llvm.minnum.v2f32(<2 x float> %a, <2 x float> %b)
store <2 x float> %val, ptr addrspace(1) %out, align 8
@@ -1774,16 +1779,17 @@ define <3 x float> @test_func_fmin_v3f32(<3 x float> %a, <3 x float> %b) nounwin
define amdgpu_kernel void @test_fmin_f16_v_ieee_on(ptr addrspace(1) %out, half %a, half %b) #0 {
; GFX8-SDAG-LABEL: test_fmin_f16_v_ieee_on:
; GFX8-SDAG: ; %bb.0:
-; GFX8-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT: s_lshr_b32 s4, s6, 16
-; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; GFX8-SDAG-NEXT: v_max_f16_e64 v0, s0, s0
+; GFX8-SDAG-NEXT: v_max_f16_e64 v1, s2, s2
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
; GFX8-SDAG-NEXT: v_min_f16_e32 v0, v1, v0
-; GFX8-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: buffer_store_short v0, off, s[4:7], 0
; GFX8-SDAG-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: test_fmin_f16_v_ieee_on:
@@ -1802,16 +1808,17 @@ define amdgpu_kernel void @test_fmin_f16_v_ieee_on(ptr addrspace(1) %out, half %
;
; GFX9-SDAG-LABEL: test_fmin_f16_v_ieee_on:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-SDAG-NEXT: s_mov_b32 s6, -1
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_lshr_b32 s4, s6, 16
-; GFX9-SDAG-NEXT: v_max_f16_e64 v0, s4, s4
-; GFX9-SDAG-NEXT: v_max_f16_e64 v1, s6, s6
+; GFX9-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX9-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; GFX9-SDAG-NEXT: v_max_f16_e64 v0, s0, s0
+; GFX9-SDAG-NEXT: v_max_f16_e64 v1, s2, s2
+; GFX9-SDAG-NEXT: s_mov_b32 s5, s1
; GFX9-SDAG-NEXT: v_min_f16_e32 v0, v1, v0
-; GFX9-SDAG-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-SDAG-NEXT: buffer_store_short v0, off, s[4:7], 0
; GFX9-SDAG-NEXT: s_endpgm
;
; GFX9-GISEL-LABEL: test_fmin_f16_v_ieee_on:
diff --git a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
index 02ce8be125afc..5f0a41f77066c 100644
--- a/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmul-2-combine-multi-use.ll
@@ -22,15 +22,15 @@ define amdgpu_kernel void @multiple_fadd_use_test_f32(ptr addrspace(1) %out, flo
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s3, -1.0
-; VI-NEXT: v_add_f32_e64 v1, s2, -1.0
-; VI-NEXT: v_cmp_gt_f32_e64 vcc, |v0|, |v1|
-; VI-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; VI-NEXT: v_add_f32_e64 v0, |v0|, |v0|
-; VI-NEXT: v_mul_f32_e32 v1, v0, v0
-; VI-NEXT: v_mad_f32 v2, -v1, v0, 1.0
+; VI-NEXT: v_add_f32_e64 v2, s3, -1.0
+; VI-NEXT: v_add_f32_e64 v3, s2, -1.0
+; VI-NEXT: v_cmp_gt_f32_e64 vcc, |v2|, |v3|
+; VI-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
+; VI-NEXT: v_add_f32_e64 v2, |v2|, |v2|
+; VI-NEXT: v_mul_f32_e32 v3, v2, v2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mad_f32 v2, -v3, v2, 1.0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -82,37 +82,37 @@ define amdgpu_kernel void @multiple_fadd_use_test_f32(ptr addrspace(1) %out, flo
define amdgpu_kernel void @multiple_use_fadd_fmac_f32(ptr addrspace(1) %out, float %x, [8 x i32], float %y) #0 {
; VI-LABEL: multiple_use_fadd_fmac_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_load_dword s3, s[8:9], 0x2c
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_add_u32 s2, s0, 4
-; VI-NEXT: v_add_f32_e64 v2, s4, s4
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_add_u32 s4, s0, 4
+; VI-NEXT: v_add_f32_e64 v2, s2, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: s_addc_u32 s3, s1, 0
+; VI-NEXT: s_addc_u32 s5, s1, 0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mac_f32_e64 v3, s4, 2.0
-; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mac_f32_e64 v3, s2, 2.0
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_endpgm
;
; GFX10-LABEL: multiple_use_fadd_fmac_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s3, s[8:9], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: ; kill: killed $sgpr8_sgpr9
; GFX10-NEXT: v_add_f32_e64 v1, s2, s2
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_fma_f32 v2, s2, 2.0, s3
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
@@ -122,13 +122,12 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f32(ptr addrspace(1) %out, flo
;
; GFX11-LABEL: multiple_use_fadd_fmac_f32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_add_f32_e64 v1, s2, s2
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_fma_f32 v2, s2, 2.0, s3
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
@@ -203,24 +202,24 @@ define amdgpu_kernel void @multiple_use_fadd_fmad_f32(ptr addrspace(1) %out, flo
define amdgpu_kernel void @multiple_use_fadd_multi_fmad_f32(ptr addrspace(1) %out, float %x, float %y, float %z) #0 {
; VI-LABEL: multiple_use_fadd_multi_fmad_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
-; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT: s_load_dword s5, s[8:9], 0x10
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_u32 s6, s4, 4
-; VI-NEXT: v_mov_b32_e32 v0, s1
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mad_f32 v2, |s0|, 2.0, v0
-; VI-NEXT: v_mad_f32 v3, |s0|, 2.0, v1
-; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v0, s3
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: s_addc_u32 s7, s5, 0
+; VI-NEXT: v_mad_f32 v2, |s2|, 2.0, v0
+; VI-NEXT: v_mad_f32 v3, |s2|, 2.0, v1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_add_u32 s4, s0, 4
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_addc_u32 s5, s1, 0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: v_mov_b32_e32 v1, s7
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: flat_store_dword v[0:1], v3
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_endpgm
@@ -228,30 +227,30 @@ define amdgpu_kernel void @multiple_use_fadd_multi_fmad_f32(ptr addrspace(1) %ou
; GFX10-LABEL: multiple_use_fadd_multi_fmad_f32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-NEXT: s_load_dword s4, s[8:9], 0x10
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_fma_f32 v1, |s0|, 2.0, s1
-; GFX10-NEXT: v_fma_f32 v2, |s0|, 2.0, s2
-; GFX10-NEXT: global_store_dword v0, v1, s[4:5]
+; GFX10-NEXT: v_fma_f32 v1, |s2|, 2.0, s3
+; GFX10-NEXT: v_fma_f32 v2, |s2|, 2.0, s4
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX10-NEXT: global_store_dword v0, v2, s[4:5] offset:4
+; GFX10-NEXT: global_store_dword v0, v2, s[0:1] offset:4
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: multiple_use_fadd_multi_fmad_f32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x10
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_fma_f32 v1, |s0|, 2.0, s1
-; GFX11-NEXT: v_fma_f32 v2, |s0|, 2.0, s2
-; GFX11-NEXT: global_store_b32 v0, v1, s[4:5] dlc
+; GFX11-NEXT: v_fma_f32 v1, |s2|, 2.0, s3
+; GFX11-NEXT: v_fma_f32 v2, |s2|, 2.0, s4
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1] dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; GFX11-NEXT: global_store_b32 v0, v2, s[4:5] offset:4 dlc
+; GFX11-NEXT: global_store_b32 v0, v2, s[0:1] offset:4 dlc
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: s_endpgm
%out.gep.1 = getelementptr float, ptr addrspace(1) %out, i32 1
@@ -267,25 +266,22 @@ define amdgpu_kernel void @multiple_use_fadd_multi_fmad_f32(ptr addrspace(1) %ou
define amdgpu_kernel void @fmul_x2_xn2_f32(ptr addrspace(1) %out, float %x, float %y) #0 {
; VI-LABEL: fmul_x2_xn2_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mul_f32_e64 v0, s2, -4.0
-; VI-NEXT: v_mul_f32_e32 v2, s2, v0
+; VI-NEXT: v_mul_f32_e64 v2, s2, -4.0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mul_f32_e32 v2, s2, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_endpgm
;
; GFX10-LABEL: fmul_x2_xn2_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mul_f32_e64 v0, s2, -4.0
@@ -296,9 +292,7 @@ define amdgpu_kernel void @fmul_x2_xn2_f32(ptr addrspace(1) %out, float %x, floa
;
; GFX11-LABEL: fmul_x2_xn2_f32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mul_f32_e64 v0, s2, -4.0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -317,26 +311,23 @@ define amdgpu_kernel void @fmul_x2_xn2_f32(ptr addrspace(1) %out, float %x, floa
define amdgpu_kernel void @fmul_x2_xn3_f32(ptr addrspace(1) %out, float %x, float %y) #0 {
; VI-LABEL: fmul_x2_xn3_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_mov_b32_e32 v0, 0xc0c00000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT: v_mov_b32_e32 v2, 0xc0c00000
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mul_f32_e32 v0, s2, v0
-; VI-NEXT: v_mul_f32_e32 v2, s2, v0
+; VI-NEXT: v_mul_f32_e32 v2, s2, v2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mul_f32_e32 v2, s2, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_endpgm
;
; GFX10-LABEL: fmul_x2_xn3_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mul_f32_e64 v0, 0xc0c00000, s2
@@ -347,9 +338,7 @@ define amdgpu_kernel void @fmul_x2_xn3_f32(ptr addrspace(1) %out, float %x, floa
;
; GFX11-LABEL: fmul_x2_xn3_f32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mul_f32_e64 v0, 0xc0c00000, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -368,72 +357,67 @@ define amdgpu_kernel void @fmul_x2_xn3_f32(ptr addrspace(1) %out, float %x, floa
define amdgpu_kernel void @multiple_fadd_use_test_f16(ptr addrspace(1) %out, i16 zeroext %x.arg, i16 zeroext %y.arg, i16 zeroext %z.arg) #0 {
; VI-DENORM-LABEL: multiple_fadd_use_test_f16:
; VI-DENORM: ; %bb.0:
-; VI-DENORM-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-DENORM-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-DENORM-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-DENORM-NEXT: s_add_i32 s12, s12, s17
; VI-DENORM-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-DENORM-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-DENORM-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DENORM-NEXT: s_lshr_b32 s3, s2, 16
-; VI-DENORM-NEXT: v_add_f16_e64 v0, s2, -1.0
-; VI-DENORM-NEXT: v_add_f16_e64 v1, s3, -1.0
-; VI-DENORM-NEXT: v_cmp_gt_f16_e64 vcc, |v1|, |v0|
-; VI-DENORM-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; VI-DENORM-NEXT: v_add_f16_e64 v0, |v0|, |v0|
-; VI-DENORM-NEXT: v_mul_f16_e32 v1, v0, v0
-; VI-DENORM-NEXT: v_fma_f16 v2, -v1, v0, 1.0
; VI-DENORM-NEXT: v_mov_b32_e32 v0, s0
+; VI-DENORM-NEXT: s_lshr_b32 s0, s2, 16
+; VI-DENORM-NEXT: v_add_f16_e64 v2, s2, -1.0
+; VI-DENORM-NEXT: v_add_f16_e64 v3, s0, -1.0
+; VI-DENORM-NEXT: v_cmp_gt_f16_e64 vcc, |v3|, |v2|
+; VI-DENORM-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; VI-DENORM-NEXT: v_add_f16_e64 v2, |v2|, |v2|
+; VI-DENORM-NEXT: v_mul_f16_e32 v3, v2, v2
; VI-DENORM-NEXT: v_mov_b32_e32 v1, s1
+; VI-DENORM-NEXT: v_fma_f16 v2, -v3, v2, 1.0
; VI-DENORM-NEXT: flat_store_short v[0:1], v2
; VI-DENORM-NEXT: s_endpgm
;
; VI-FLUSH-LABEL: multiple_fadd_use_test_f16:
; VI-FLUSH: ; %bb.0:
-; VI-FLUSH-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-FLUSH-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-FLUSH-NEXT: s_add_i32 s12, s12, s17
; VI-FLUSH-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-FLUSH-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; VI-FLUSH-NEXT: s_lshr_b32 s3, s2, 16
-; VI-FLUSH-NEXT: v_add_f16_e64 v0, s2, -1.0
-; VI-FLUSH-NEXT: v_add_f16_e64 v1, s3, -1.0
-; VI-FLUSH-NEXT: v_cmp_gt_f16_e64 vcc, |v1|, |v0|
-; VI-FLUSH-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; VI-FLUSH-NEXT: v_add_f16_e64 v0, |v0|, |v0|
-; VI-FLUSH-NEXT: v_mul_f16_e32 v1, v0, v0
-; VI-FLUSH-NEXT: v_mad_f16 v2, -v1, v0, 1.0
; VI-FLUSH-NEXT: v_mov_b32_e32 v0, s0
+; VI-FLUSH-NEXT: s_lshr_b32 s0, s2, 16
+; VI-FLUSH-NEXT: v_add_f16_e64 v2, s2, -1.0
+; VI-FLUSH-NEXT: v_add_f16_e64 v3, s0, -1.0
+; VI-FLUSH-NEXT: v_cmp_gt_f16_e64 vcc, |v3|, |v2|
+; VI-FLUSH-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; VI-FLUSH-NEXT: v_add_f16_e64 v2, |v2|, |v2|
+; VI-FLUSH-NEXT: v_mul_f16_e32 v3, v2, v2
; VI-FLUSH-NEXT: v_mov_b32_e32 v1, s1
+; VI-FLUSH-NEXT: v_mad_f16 v2, -v3, v2, 1.0
; VI-FLUSH-NEXT: flat_store_short v[0:1], v2
; VI-FLUSH-NEXT: s_endpgm
;
; GFX10-DENORM-LABEL: multiple_fadd_use_test_f16:
; GFX10-DENORM: ; %bb.0:
-; GFX10-DENORM-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-DENORM-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-DENORM-NEXT: v_mov_b32_e32 v2, 0
; GFX10-DENORM-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-DENORM-NEXT: s_lshr_b32 s1, s0, 16
-; GFX10-DENORM-NEXT: v_add_f16_e64 v0, s0, -1.0
-; GFX10-DENORM-NEXT: v_add_f16_e64 v1, s1, -1.0
-; GFX10-DENORM-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-DENORM-NEXT: s_lshr_b32 s3, s2, 16
+; GFX10-DENORM-NEXT: v_add_f16_e64 v0, s2, -1.0
+; GFX10-DENORM-NEXT: v_add_f16_e64 v1, s3, -1.0
; GFX10-DENORM-NEXT: v_cmp_gt_f16_e64 vcc_lo, |v1|, |v0|
; GFX10-DENORM-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
; GFX10-DENORM-NEXT: v_add_f16_e64 v0, |v0|, |v0|
; GFX10-DENORM-NEXT: v_mul_f16_e32 v1, v0, v0
; GFX10-DENORM-NEXT: v_fma_f16 v0, -v1, v0, 1.0
-; GFX10-DENORM-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DENORM-NEXT: global_store_short v2, v0, s[0:1]
; GFX10-DENORM-NEXT: s_endpgm
;
; GFX10-FLUSH-LABEL: multiple_fadd_use_test_f16:
; GFX10-FLUSH: ; %bb.0:
-; GFX10-FLUSH-NEXT: s_load_dword s0, s[8:9], 0x8
+; GFX10-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-FLUSH-NEXT: s_lshr_b32 s1, s0, 16
-; GFX10-FLUSH-NEXT: v_add_f16_e64 v0, s0, -1.0
-; GFX10-FLUSH-NEXT: v_add_f16_e64 v1, s1, -1.0
-; GFX10-FLUSH-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-FLUSH-NEXT: s_lshr_b32 s3, s2, 16
+; GFX10-FLUSH-NEXT: v_add_f16_e64 v0, s2, -1.0
+; GFX10-FLUSH-NEXT: v_add_f16_e64 v1, s3, -1.0
; GFX10-FLUSH-NEXT: v_cmp_gt_f16_e64 vcc_lo, |v1|, |v0|
; GFX10-FLUSH-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
; GFX10-FLUSH-NEXT: v_add_f16_e64 v0, |v0|, |v0|
@@ -441,19 +425,17 @@ define amdgpu_kernel void @multiple_fadd_use_test_f16(ptr addrspace(1) %out, i16
; GFX10-FLUSH-NEXT: v_mul_f16_e32 v0, v1, v0
; GFX10-FLUSH-NEXT: v_mov_b32_e32 v1, 0
; GFX10-FLUSH-NEXT: v_sub_f16_e32 v0, 1.0, v0
-; GFX10-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-FLUSH-NEXT: global_store_short v1, v0, s[0:1]
; GFX10-FLUSH-NEXT: s_endpgm
;
; GFX11-DENORM-TRUE16-LABEL: multiple_fadd_use_test_f16:
; GFX11-DENORM-TRUE16: ; %bb.0:
-; GFX11-DENORM-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-DENORM-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-DENORM-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-DENORM-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-DENORM-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-DENORM-TRUE16-NEXT: v_add_f16_e64 v0.h, s0, -1.0
-; GFX11-DENORM-TRUE16-NEXT: v_add_f16_e64 v0.l, s1, -1.0
-; GFX11-DENORM-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-DENORM-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-DENORM-TRUE16-NEXT: v_add_f16_e64 v0.h, s2, -1.0
+; GFX11-DENORM-TRUE16-NEXT: v_add_f16_e64 v0.l, s3, -1.0
; GFX11-DENORM-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DENORM-TRUE16-NEXT: v_cmp_gt_f16_e64 s2, |v0.l|, |v0.h|
; GFX11-DENORM-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s2
@@ -462,19 +444,17 @@ define amdgpu_kernel void @multiple_fadd_use_test_f16(ptr addrspace(1) %out, i16
; GFX11-DENORM-TRUE16-NEXT: v_mul_f16_e32 v0.h, v0.l, v0.l
; GFX11-DENORM-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-DENORM-TRUE16-NEXT: v_fma_f16 v0.l, -v0.h, v0.l, 1.0
-; GFX11-DENORM-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DENORM-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-DENORM-TRUE16-NEXT: s_endpgm
;
; GFX11-DENORM-FAKE16-LABEL: multiple_fadd_use_test_f16:
; GFX11-DENORM-FAKE16: ; %bb.0:
-; GFX11-DENORM-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-DENORM-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-DENORM-FAKE16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-DENORM-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-DENORM-FAKE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-DENORM-FAKE16-NEXT: v_add_f16_e64 v0, s0, -1.0
-; GFX11-DENORM-FAKE16-NEXT: v_add_f16_e64 v1, s1, -1.0
-; GFX11-DENORM-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-DENORM-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-DENORM-FAKE16-NEXT: v_add_f16_e64 v0, s2, -1.0
+; GFX11-DENORM-FAKE16-NEXT: v_add_f16_e64 v1, s3, -1.0
; GFX11-DENORM-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-DENORM-FAKE16-NEXT: v_cmp_gt_f16_e64 vcc_lo, |v1|, |v0|
; GFX11-DENORM-FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
@@ -482,40 +462,36 @@ define amdgpu_kernel void @multiple_fadd_use_test_f16(ptr addrspace(1) %out, i16
; GFX11-DENORM-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DENORM-FAKE16-NEXT: v_mul_f16_e32 v1, v0, v0
; GFX11-DENORM-FAKE16-NEXT: v_fma_f16 v0, -v1, v0, 1.0
-; GFX11-DENORM-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DENORM-FAKE16-NEXT: global_store_b16 v2, v0, s[0:1]
; GFX11-DENORM-FAKE16-NEXT: s_endpgm
;
; GFX11-FLUSH-TRUE16-LABEL: multiple_fadd_use_test_f16:
; GFX11-FLUSH-TRUE16: ; %bb.0:
-; GFX11-FLUSH-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-FLUSH-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FLUSH-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-FLUSH-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FLUSH-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e64 v0.h, s0, -1.0
-; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e64 v0.l, s1, -1.0
+; GFX11-FLUSH-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e64 v0.h, s2, -1.0
+; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e64 v0.l, s3, -1.0
; GFX11-FLUSH-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FLUSH-TRUE16-NEXT: v_cmp_gt_f16_e64 s0, |v0.l|, |v0.h|
-; GFX11-FLUSH-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s0
-; GFX11-FLUSH-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FLUSH-TRUE16-NEXT: v_cmp_gt_f16_e64 s2, |v0.l|, |v0.h|
+; GFX11-FLUSH-TRUE16-NEXT: v_cndmask_b16 v0.l, v0.h, v0.l, s2
; GFX11-FLUSH-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e64 v0.l, |v0.l|, |v0.l|
; GFX11-FLUSH-TRUE16-NEXT: v_mul_f16_e32 v0.h, v0.l, v0.l
; GFX11-FLUSH-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FLUSH-TRUE16-NEXT: v_mul_f16_e32 v0.l, v0.h, v0.l
; GFX11-FLUSH-TRUE16-NEXT: v_sub_f16_e32 v0.l, 1.0, v0.l
-; GFX11-FLUSH-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FLUSH-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-FLUSH-TRUE16-NEXT: s_endpgm
;
; GFX11-FLUSH-FAKE16-LABEL: multiple_fadd_use_test_f16:
; GFX11-FLUSH-FAKE16: ; %bb.0:
-; GFX11-FLUSH-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-FLUSH-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FLUSH-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FLUSH-FAKE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e64 v0, s0, -1.0
-; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e64 v1, s1, -1.0
-; GFX11-FLUSH-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FLUSH-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e64 v0, s2, -1.0
+; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e64 v1, s3, -1.0
; GFX11-FLUSH-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-FLUSH-FAKE16-NEXT: v_cmp_gt_f16_e64 vcc_lo, |v1|, |v0|
; GFX11-FLUSH-FAKE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
@@ -526,7 +502,6 @@ define amdgpu_kernel void @multiple_fadd_use_test_f16(ptr addrspace(1) %out, i16
; GFX11-FLUSH-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-FLUSH-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FLUSH-FAKE16-NEXT: v_sub_f16_e32 v0, 1.0, v0
-; GFX11-FLUSH-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FLUSH-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX11-FLUSH-FAKE16-NEXT: s_endpgm
%x = bitcast i16 %x.arg to half
@@ -549,57 +524,53 @@ define amdgpu_kernel void @multiple_fadd_use_test_f16(ptr addrspace(1) %out, i16
define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16 zeroext %x.arg, i16 zeroext %y.arg) #0 {
; VI-DENORM-LABEL: multiple_use_fadd_fmac_f16:
; VI-DENORM: ; %bb.0:
-; VI-DENORM-NEXT: s_load_dword s4, s[8:9], 0x8
-; VI-DENORM-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-DENORM-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-DENORM-NEXT: s_add_i32 s12, s12, s17
; VI-DENORM-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-DENORM-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-DENORM-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DENORM-NEXT: s_lshr_b32 s3, s4, 16
-; VI-DENORM-NEXT: v_mov_b32_e32 v0, s3
-; VI-DENORM-NEXT: v_fma_f16 v3, s4, 2.0, v0
+; VI-DENORM-NEXT: s_lshr_b32 s3, s2, 16
; VI-DENORM-NEXT: v_mov_b32_e32 v0, s0
-; VI-DENORM-NEXT: v_add_f16_e64 v2, s4, s4
-; VI-DENORM-NEXT: s_add_u32 s2, s0, 2
+; VI-DENORM-NEXT: v_add_f16_e64 v2, s2, s2
; VI-DENORM-NEXT: v_mov_b32_e32 v1, s1
-; VI-DENORM-NEXT: s_addc_u32 s3, s1, 0
+; VI-DENORM-NEXT: s_add_u32 s0, s0, 2
+; VI-DENORM-NEXT: v_mov_b32_e32 v3, s3
+; VI-DENORM-NEXT: s_addc_u32 s1, s1, 0
; VI-DENORM-NEXT: flat_store_short v[0:1], v2
; VI-DENORM-NEXT: s_waitcnt vmcnt(0)
-; VI-DENORM-NEXT: v_mov_b32_e32 v0, s2
-; VI-DENORM-NEXT: v_mov_b32_e32 v1, s3
+; VI-DENORM-NEXT: v_mov_b32_e32 v0, s0
+; VI-DENORM-NEXT: v_fma_f16 v3, s2, 2.0, v3
+; VI-DENORM-NEXT: v_mov_b32_e32 v1, s1
; VI-DENORM-NEXT: flat_store_short v[0:1], v3
; VI-DENORM-NEXT: s_waitcnt vmcnt(0)
; VI-DENORM-NEXT: s_endpgm
;
; VI-FLUSH-LABEL: multiple_use_fadd_fmac_f16:
; VI-FLUSH: ; %bb.0:
-; VI-FLUSH-NEXT: s_load_dword s4, s[8:9], 0x8
-; VI-FLUSH-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-FLUSH-NEXT: s_add_i32 s12, s12, s17
; VI-FLUSH-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-FLUSH-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; VI-FLUSH-NEXT: s_lshr_b32 s3, s4, 16
+; VI-FLUSH-NEXT: s_lshr_b32 s3, s2, 16
; VI-FLUSH-NEXT: v_mov_b32_e32 v0, s0
-; VI-FLUSH-NEXT: v_add_f16_e64 v2, s4, s4
-; VI-FLUSH-NEXT: s_add_u32 s2, s0, 2
+; VI-FLUSH-NEXT: v_add_f16_e64 v2, s2, s2
; VI-FLUSH-NEXT: v_mov_b32_e32 v1, s1
+; VI-FLUSH-NEXT: s_add_u32 s0, s0, 2
; VI-FLUSH-NEXT: v_mov_b32_e32 v3, s3
-; VI-FLUSH-NEXT: s_addc_u32 s3, s1, 0
+; VI-FLUSH-NEXT: s_addc_u32 s1, s1, 0
; VI-FLUSH-NEXT: flat_store_short v[0:1], v2
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; VI-FLUSH-NEXT: v_mov_b32_e32 v0, s2
-; VI-FLUSH-NEXT: v_mac_f16_e64 v3, s4, 2.0
-; VI-FLUSH-NEXT: v_mov_b32_e32 v1, s3
+; VI-FLUSH-NEXT: v_mov_b32_e32 v0, s0
+; VI-FLUSH-NEXT: v_mac_f16_e64 v3, s2, 2.0
+; VI-FLUSH-NEXT: v_mov_b32_e32 v1, s1
; VI-FLUSH-NEXT: flat_store_short v[0:1], v3
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
; VI-FLUSH-NEXT: s_endpgm
;
; GFX10-DENORM-LABEL: multiple_use_fadd_fmac_f16:
; GFX10-DENORM: ; %bb.0:
-; GFX10-DENORM-NEXT: s_clause 0x1
-; GFX10-DENORM-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-DENORM-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-DENORM-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-DENORM-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DENORM-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DENORM-NEXT: s_lshr_b32 s3, s2, 16
@@ -613,9 +584,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
;
; GFX10-FLUSH-LABEL: multiple_use_fadd_fmac_f16:
; GFX10-FLUSH: ; %bb.0:
-; GFX10-FLUSH-NEXT: s_clause 0x1
-; GFX10-FLUSH-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-FLUSH-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-FLUSH-NEXT: v_mov_b32_e32 v1, 0
; GFX10-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-FLUSH-NEXT: v_add_f16_e64 v0, s2, s2
@@ -629,9 +598,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
;
; GFX11-DENORM-TRUE16-LABEL: multiple_use_fadd_fmac_f16:
; GFX11-DENORM-TRUE16: ; %bb.0:
-; GFX11-DENORM-TRUE16-NEXT: s_clause 0x1
-; GFX11-DENORM-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-DENORM-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-DENORM-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-DENORM-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-DENORM-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DENORM-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -645,9 +612,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
;
; GFX11-DENORM-FAKE16-LABEL: multiple_use_fadd_fmac_f16:
; GFX11-DENORM-FAKE16: ; %bb.0:
-; GFX11-DENORM-FAKE16-NEXT: s_clause 0x1
-; GFX11-DENORM-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-DENORM-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-DENORM-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-DENORM-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-DENORM-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DENORM-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -661,9 +626,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
;
; GFX11-FLUSH-TRUE16-LABEL: multiple_use_fadd_fmac_f16:
; GFX11-FLUSH-TRUE16: ; %bb.0:
-; GFX11-FLUSH-TRUE16-NEXT: s_clause 0x1
-; GFX11-FLUSH-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FLUSH-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FLUSH-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FLUSH-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-FLUSH-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, s2
@@ -678,9 +641,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
;
; GFX11-FLUSH-FAKE16-LABEL: multiple_use_fadd_fmac_f16:
; GFX11-FLUSH-FAKE16: ; %bb.0:
-; GFX11-FLUSH-FAKE16-NEXT: s_clause 0x1
-; GFX11-FLUSH-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FLUSH-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FLUSH-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FLUSH-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-FLUSH-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e64 v0, s2, s2
@@ -705,57 +666,53 @@ define amdgpu_kernel void @multiple_use_fadd_fmac_f16(ptr addrspace(1) %out, i16
define amdgpu_kernel void @multiple_use_fadd_fmad_f16(ptr addrspace(1) %out, i16 zeroext %x.arg, i16 zeroext %y.arg) #0 {
; VI-DENORM-LABEL: multiple_use_fadd_fmad_f16:
; VI-DENORM: ; %bb.0:
-; VI-DENORM-NEXT: s_load_dword s4, s[8:9], 0x8
-; VI-DENORM-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-DENORM-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-DENORM-NEXT: s_add_i32 s12, s12, s17
; VI-DENORM-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-DENORM-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-DENORM-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DENORM-NEXT: s_lshr_b32 s3, s4, 16
-; VI-DENORM-NEXT: v_mov_b32_e32 v0, s3
-; VI-DENORM-NEXT: v_fma_f16 v3, |s4|, 2.0, v0
+; VI-DENORM-NEXT: s_lshr_b32 s3, s2, 16
; VI-DENORM-NEXT: v_mov_b32_e32 v0, s0
-; VI-DENORM-NEXT: v_add_f16_e64 v2, |s4|, |s4|
-; VI-DENORM-NEXT: s_add_u32 s2, s0, 2
+; VI-DENORM-NEXT: v_add_f16_e64 v2, |s2|, |s2|
; VI-DENORM-NEXT: v_mov_b32_e32 v1, s1
-; VI-DENORM-NEXT: s_addc_u32 s3, s1, 0
+; VI-DENORM-NEXT: s_add_u32 s0, s0, 2
+; VI-DENORM-NEXT: v_mov_b32_e32 v3, s3
+; VI-DENORM-NEXT: s_addc_u32 s1, s1, 0
; VI-DENORM-NEXT: flat_store_short v[0:1], v2
; VI-DENORM-NEXT: s_waitcnt vmcnt(0)
-; VI-DENORM-NEXT: v_mov_b32_e32 v0, s2
-; VI-DENORM-NEXT: v_mov_b32_e32 v1, s3
+; VI-DENORM-NEXT: v_mov_b32_e32 v0, s0
+; VI-DENORM-NEXT: v_fma_f16 v3, |s2|, 2.0, v3
+; VI-DENORM-NEXT: v_mov_b32_e32 v1, s1
; VI-DENORM-NEXT: flat_store_short v[0:1], v3
; VI-DENORM-NEXT: s_waitcnt vmcnt(0)
; VI-DENORM-NEXT: s_endpgm
;
; VI-FLUSH-LABEL: multiple_use_fadd_fmad_f16:
; VI-FLUSH: ; %bb.0:
-; VI-FLUSH-NEXT: s_load_dword s4, s[8:9], 0x8
-; VI-FLUSH-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-FLUSH-NEXT: s_add_i32 s12, s12, s17
; VI-FLUSH-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-FLUSH-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
-; VI-FLUSH-NEXT: s_lshr_b32 s3, s4, 16
-; VI-FLUSH-NEXT: v_mov_b32_e32 v0, s3
-; VI-FLUSH-NEXT: v_mad_f16 v3, |s4|, 2.0, v0
+; VI-FLUSH-NEXT: s_lshr_b32 s3, s2, 16
; VI-FLUSH-NEXT: v_mov_b32_e32 v0, s0
-; VI-FLUSH-NEXT: v_add_f16_e64 v2, |s4|, |s4|
-; VI-FLUSH-NEXT: s_add_u32 s2, s0, 2
+; VI-FLUSH-NEXT: v_add_f16_e64 v2, |s2|, |s2|
; VI-FLUSH-NEXT: v_mov_b32_e32 v1, s1
-; VI-FLUSH-NEXT: s_addc_u32 s3, s1, 0
+; VI-FLUSH-NEXT: s_add_u32 s0, s0, 2
+; VI-FLUSH-NEXT: v_mov_b32_e32 v3, s3
+; VI-FLUSH-NEXT: s_addc_u32 s1, s1, 0
; VI-FLUSH-NEXT: flat_store_short v[0:1], v2
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
-; VI-FLUSH-NEXT: v_mov_b32_e32 v0, s2
-; VI-FLUSH-NEXT: v_mov_b32_e32 v1, s3
+; VI-FLUSH-NEXT: v_mov_b32_e32 v0, s0
+; VI-FLUSH-NEXT: v_mad_f16 v3, |s2|, 2.0, v3
+; VI-FLUSH-NEXT: v_mov_b32_e32 v1, s1
; VI-FLUSH-NEXT: flat_store_short v[0:1], v3
; VI-FLUSH-NEXT: s_waitcnt vmcnt(0)
; VI-FLUSH-NEXT: s_endpgm
;
; GFX10-DENORM-LABEL: multiple_use_fadd_fmad_f16:
; GFX10-DENORM: ; %bb.0:
-; GFX10-DENORM-NEXT: s_clause 0x1
-; GFX10-DENORM-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-DENORM-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-DENORM-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-DENORM-NEXT: v_mov_b32_e32 v0, 0
; GFX10-DENORM-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-DENORM-NEXT: s_lshr_b32 s3, s2, 16
@@ -769,9 +726,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmad_f16(ptr addrspace(1) %out, i16
;
; GFX10-FLUSH-LABEL: multiple_use_fadd_fmad_f16:
; GFX10-FLUSH: ; %bb.0:
-; GFX10-FLUSH-NEXT: s_clause 0x1
-; GFX10-FLUSH-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-FLUSH-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-FLUSH-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-FLUSH-NEXT: v_mov_b32_e32 v1, 0
; GFX10-FLUSH-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-FLUSH-NEXT: v_add_f16_e64 v0, |s2|, |s2|
@@ -785,9 +740,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmad_f16(ptr addrspace(1) %out, i16
;
; GFX11-DENORM-TRUE16-LABEL: multiple_use_fadd_fmad_f16:
; GFX11-DENORM-TRUE16: ; %bb.0:
-; GFX11-DENORM-TRUE16-NEXT: s_clause 0x1
-; GFX11-DENORM-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-DENORM-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-DENORM-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-DENORM-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-DENORM-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DENORM-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -801,9 +754,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmad_f16(ptr addrspace(1) %out, i16
;
; GFX11-DENORM-FAKE16-LABEL: multiple_use_fadd_fmad_f16:
; GFX11-DENORM-FAKE16: ; %bb.0:
-; GFX11-DENORM-FAKE16-NEXT: s_clause 0x1
-; GFX11-DENORM-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-DENORM-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-DENORM-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-DENORM-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-DENORM-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DENORM-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -817,9 +768,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmad_f16(ptr addrspace(1) %out, i16
;
; GFX11-FLUSH-TRUE16-LABEL: multiple_use_fadd_fmad_f16:
; GFX11-FLUSH-TRUE16: ; %bb.0:
-; GFX11-FLUSH-TRUE16-NEXT: s_clause 0x1
-; GFX11-FLUSH-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FLUSH-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FLUSH-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FLUSH-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-FLUSH-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FLUSH-TRUE16-NEXT: v_add_f16_e64 v0.l, |s2|, |s2|
@@ -834,9 +783,7 @@ define amdgpu_kernel void @multiple_use_fadd_fmad_f16(ptr addrspace(1) %out, i16
;
; GFX11-FLUSH-FAKE16-LABEL: multiple_use_fadd_fmad_f16:
; GFX11-FLUSH-FAKE16: ; %bb.0:
-; GFX11-FLUSH-FAKE16-NEXT: s_clause 0x1
-; GFX11-FLUSH-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FLUSH-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FLUSH-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FLUSH-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-FLUSH-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FLUSH-FAKE16-NEXT: v_add_f16_e64 v0, |s2|, |s2|
@@ -1012,25 +959,22 @@ define amdgpu_kernel void @multiple_use_fadd_multi_fmad_f16(ptr addrspace(1) %ou
define amdgpu_kernel void @fmul_x2_xn2_f16(ptr addrspace(1) %out, i16 zeroext %x.arg, i16 zeroext %y.arg) #0 {
; VI-LABEL: fmul_x2_xn2_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mul_f16_e64 v0, s2, -4.0
-; VI-NEXT: v_mul_f16_e32 v2, s2, v0
+; VI-NEXT: v_mul_f16_e64 v2, s2, -4.0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mul_f16_e32 v2, s2, v2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_endpgm
;
; GFX10-LABEL: fmul_x2_xn2_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mul_f16_e64 v0, s2, -4.0
@@ -1041,9 +985,7 @@ define amdgpu_kernel void @fmul_x2_xn2_f16(ptr addrspace(1) %out, i16 zeroext %x
;
; GFX11-TRUE16-LABEL: fmul_x2_xn2_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mul_f16_e64 v0.l, s2, -4.0
@@ -1055,9 +997,7 @@ define amdgpu_kernel void @fmul_x2_xn2_f16(ptr addrspace(1) %out, i16 zeroext %x
;
; GFX11-FAKE16-LABEL: fmul_x2_xn2_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mul_f16_e64 v0, s2, -4.0
@@ -1079,26 +1019,23 @@ define amdgpu_kernel void @fmul_x2_xn2_f16(ptr addrspace(1) %out, i16 zeroext %x
define amdgpu_kernel void @fmul_x2_xn3_f16(ptr addrspace(1) %out, i16 zeroext %x.arg, i16 zeroext %y.arg) #0 {
; VI-LABEL: fmul_x2_xn3_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_mov_b32_e32 v0, 0xc600
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT: v_mov_b32_e32 v2, 0xc600
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mul_f16_e32 v0, s2, v0
-; VI-NEXT: v_mul_f16_e32 v2, s2, v0
+; VI-NEXT: v_mul_f16_e32 v2, s2, v2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mul_f16_e32 v2, s2, v2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_endpgm
;
; GFX10-LABEL: fmul_x2_xn3_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mul_f16_e64 v0, 0xc600, s2
@@ -1109,9 +1046,7 @@ define amdgpu_kernel void @fmul_x2_xn3_f16(ptr addrspace(1) %out, i16 zeroext %x
;
; GFX11-TRUE16-LABEL: fmul_x2_xn3_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_mul_f16_e64 v0.l, 0xc600, s2
@@ -1123,9 +1058,7 @@ define amdgpu_kernel void @fmul_x2_xn3_f16(ptr addrspace(1) %out, i16 zeroext %x
;
; GFX11-FAKE16-LABEL: fmul_x2_xn3_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mul_f16_e64 v0, 0xc600, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fnearbyint.ll b/llvm/test/CodeGen/AMDGPU/fnearbyint.ll
index 6c2ab5fb15a20..2fa8981ee9416 100644
--- a/llvm/test/CodeGen/AMDGPU/fnearbyint.ll
+++ b/llvm/test/CodeGen/AMDGPU/fnearbyint.ll
@@ -16,47 +16,43 @@ declare <4 x double> @llvm.nearbyint.v4f64(<4 x double>) #0
define amdgpu_kernel void @fnearbyint_f16(ptr addrspace(1) %out, half %in) #1 {
; SI-LABEL: fnearbyint_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s6
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: v_rndne_f32_e32 v0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; CI-LABEL: fnearbyint_f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[4:5], 0xb
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s2
; CI-NEXT: s_mov_b32 s2, -1
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; CI-NEXT: v_rndne_f32_e32 v0, v0
; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: buffer_store_short v0, off, s[0:3], 0
; CI-NEXT: s_endpgm
;
; VI-LABEL: fnearbyint_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_rndne_f16_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_rndne_f16_e32 v2, s2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: fnearbyint_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_rndne_f16_e32 v0.l, s2
@@ -65,9 +61,7 @@ define amdgpu_kernel void @fnearbyint_f16(ptr addrspace(1) %out, half %in) #1 {
;
; GFX11-FAKE16-LABEL: fnearbyint_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_rndne_f16_e32 v1, s2
@@ -81,31 +75,29 @@ define amdgpu_kernel void @fnearbyint_f16(ptr addrspace(1) %out, half %in) #1 {
define amdgpu_kernel void @fnearbyint_f32(ptr addrspace(1) %out, float %in) #1 {
; SICI-LABEL: fnearbyint_f32:
; SICI: ; %bb.0: ; %entry
-; SICI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SICI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SICI-NEXT: s_mov_b32 s3, 0xf000
-; SICI-NEXT: s_mov_b32 s2, -1
+; SICI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SICI-NEXT: s_mov_b32 s7, 0xf000
+; SICI-NEXT: s_mov_b32 s6, -1
; SICI-NEXT: s_waitcnt lgkmcnt(0)
-; SICI-NEXT: v_rndne_f32_e32 v0, s6
-; SICI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SICI-NEXT: s_mov_b32 s4, s0
+; SICI-NEXT: s_mov_b32 s5, s1
+; SICI-NEXT: v_rndne_f32_e32 v0, s2
+; SICI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SICI-NEXT: s_endpgm
;
; VI-LABEL: fnearbyint_f32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_rndne_f32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_rndne_f32_e32 v2, s2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: fnearbyint_f32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_rndne_f32_e32 v1, s2
@@ -121,24 +113,25 @@ define amdgpu_kernel void @fnearbyint_v2f32(ptr addrspace(1) %out, <2 x float> %
; SICI-LABEL: fnearbyint_v2f32:
; SICI: ; %bb.0: ; %entry
; SICI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SICI-NEXT: s_mov_b32 s7, 0xf000
+; SICI-NEXT: s_mov_b32 s6, -1
; SICI-NEXT: s_waitcnt lgkmcnt(0)
-; SICI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SICI-NEXT: s_mov_b32 s3, 0xf000
-; SICI-NEXT: s_mov_b32 s2, -1
-; SICI-NEXT: v_rndne_f32_e32 v1, s5
-; SICI-NEXT: v_rndne_f32_e32 v0, s4
-; SICI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SICI-NEXT: s_mov_b32 s4, s0
+; SICI-NEXT: s_mov_b32 s5, s1
+; SICI-NEXT: v_rndne_f32_e32 v1, s3
+; SICI-NEXT: v_rndne_f32_e32 v0, s2
+; SICI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SICI-NEXT: s_endpgm
;
; VI-LABEL: fnearbyint_v2f32:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_rndne_f32_e32 v1, s3
-; VI-NEXT: v_rndne_f32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_rndne_f32_e32 v3, s3
+; VI-NEXT: v_rndne_f32_e32 v2, s2
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: fnearbyint_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
index 3cbde439cdf99..f2fc2626a9f36 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-combines.new.ll
@@ -4027,12 +4027,12 @@ define amdgpu_kernel void @s_fneg_select_infloop_regression_f32(float %arg, i1 %
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_bitcmp1_b32 s1, 0
-; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v2, s0
; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; SI-NEXT: v_cndmask_b32_e64 v2, -v0, 0, s[0:1]
+; SI-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[0:1]
; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: v_mov_b32_e32 v1, s3
+; SI-NEXT: v_cndmask_b32_e64 v2, -v2, 0, s[0:1]
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_endpgm
;
@@ -4041,12 +4041,12 @@ define amdgpu_kernel void @s_fneg_select_infloop_regression_f32(float %arg, i1 %
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_bitcmp1_b32 s1, 0
-; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; VI-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; VI-NEXT: v_cndmask_b32_e64 v2, -v0, 0, s[0:1]
+; VI-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[0:1]
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_cndmask_b32_e64 v2, -v2, 0, s[0:1]
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%i = select i1 %arg1, float 0.0, float %arg
@@ -4241,41 +4241,39 @@ define float @v_fneg_select_infloop_regression_neg_inline_imm_f32_commute2(float
define amdgpu_kernel void @s_fneg_select_infloop_regression_f64(double %arg, i1 %arg1, ptr addrspace(1) %ptr) {
; SI-LABEL: s_fneg_select_infloop_regression_f64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; SI-NEXT: v_bfrev_b32_e32 v0, 1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_bitcmp1_b32 s6, 0
-; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_bitcmp1_b32 s2, 0
+; SI-NEXT: s_cselect_b64 s[2:3], -1, 0
; SI-NEXT: v_mov_b32_e32 v1, s1
-; SI-NEXT: s_and_b64 s[6:7], s[4:5], exec
-; SI-NEXT: v_cndmask_b32_e64 v0, -v1, v0, s[4:5]
+; SI-NEXT: s_and_b64 s[6:7], s[2:3], exec
+; SI-NEXT: v_cndmask_b32_e64 v0, -v1, v0, s[2:3]
; SI-NEXT: s_cselect_b32 s0, 0, s0
-; SI-NEXT: v_mov_b32_e32 v2, s2
-; SI-NEXT: v_cndmask_b32_e64 v1, v0, 0, s[4:5]
+; SI-NEXT: v_mov_b32_e32 v2, s4
+; SI-NEXT: v_cndmask_b32_e64 v1, v0, 0, s[2:3]
; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v3, s3
+; SI-NEXT: v_mov_b32_e32 v3, s5
; SI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_fneg_select_infloop_regression_f64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: v_bfrev_b32_e32 v0, 1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitcmp1_b32 s6, 0
-; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; VI-NEXT: s_bitcmp1_b32 s2, 0
+; VI-NEXT: s_cselect_b64 s[2:3], -1, 0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: s_and_b64 s[6:7], s[4:5], exec
-; VI-NEXT: v_cndmask_b32_e64 v0, -v1, v0, s[4:5]
+; VI-NEXT: s_and_b64 s[6:7], s[2:3], exec
+; VI-NEXT: v_cndmask_b32_e64 v0, -v1, v0, s[2:3]
; VI-NEXT: s_cselect_b32 s0, 0, s0
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: v_cndmask_b32_e64 v1, v0, 0, s[4:5]
+; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_cndmask_b32_e64 v1, v0, 0, s[2:3]
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
%i = select i1 %arg1, double 0.0, double %arg
@@ -4489,12 +4487,12 @@ define amdgpu_kernel void @s_fabs_select_infloop_regression_f32(float %arg, i1 %
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_bitcmp1_b32 s1, 0
-; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v2, s0
; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; SI-NEXT: v_cndmask_b32_e64 v2, |v0|, 0, s[0:1]
+; SI-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[0:1]
; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: v_mov_b32_e32 v1, s3
+; SI-NEXT: v_cndmask_b32_e64 v2, |v2|, 0, s[0:1]
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_endpgm
;
@@ -4503,12 +4501,12 @@ define amdgpu_kernel void @s_fabs_select_infloop_regression_f32(float %arg, i1 %
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_bitcmp1_b32 s1, 0
-; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; VI-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; VI-NEXT: v_cndmask_b32_e64 v2, |v0|, 0, s[0:1]
+; VI-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[0:1]
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_cndmask_b32_e64 v2, |v2|, 0, s[0:1]
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%i = select i1 %arg1, float 0.0, float %arg
@@ -4539,12 +4537,12 @@ define amdgpu_kernel void @s_fneg_fabs_select_infloop_regression(float %arg, i1
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_bitcmp1_b32 s1, 0
-; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v2, s0
; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; SI-NEXT: v_cndmask_b32_e64 v2, -|v0|, 0, s[0:1]
+; SI-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[0:1]
; SI-NEXT: v_mov_b32_e32 v0, s2
; SI-NEXT: v_mov_b32_e32 v1, s3
+; SI-NEXT: v_cndmask_b32_e64 v2, -|v2|, 0, s[0:1]
; SI-NEXT: flat_store_dword v[0:1], v2
; SI-NEXT: s_endpgm
;
@@ -4553,12 +4551,12 @@ define amdgpu_kernel void @s_fneg_fabs_select_infloop_regression(float %arg, i1
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_bitcmp1_b32 s1, 0
-; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
-; VI-NEXT: v_cndmask_b32_e64 v0, v0, 0, s[0:1]
-; VI-NEXT: v_cndmask_b32_e64 v2, -|v0|, 0, s[0:1]
+; VI-NEXT: v_cndmask_b32_e64 v2, v2, 0, s[0:1]
; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: v_cndmask_b32_e64 v2, -|v2|, 0, s[0:1]
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%i = select i1 %arg1, float 0.0, float %arg
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
index 5424ebfcffcd1..f241afe7b37ca 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.bf16.ll
@@ -8,52 +8,49 @@
define amdgpu_kernel void @fneg_fabs_fadd_bf16(ptr addrspace(1) %out, bfloat %x, bfloat %y) {
; CI-LABEL: fneg_fabs_fadd_bf16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s3, s2, 0x7fff
-; CI-NEXT: s_lshl_b32 s3, s3, 16
-; CI-NEXT: s_and_b32 s2, s2, 0xffff0000
-; CI-NEXT: v_mov_b32_e32 v0, s3
-; CI-NEXT: v_sub_f32_e32 v0, s2, v0
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0x7fff
+; CI-NEXT: s_lshl_b32 s0, s0, 16
; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: s_and_b32 s1, s2, 0xffff0000
+; CI-NEXT: v_mov_b32_e32 v2, s0
+; CI-NEXT: v_sub_f32_e32 v2, s1, v2
+; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; CI-NEXT: flat_store_short v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: fneg_fabs_fadd_bf16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s3, s2, 0x7fff
-; VI-NEXT: s_lshl_b32 s3, s3, 16
-; VI-NEXT: s_and_b32 s2, s2, 0xffff0000
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_sub_f32_e32 v0, s2, v0
-; VI-NEXT: v_bfe_u32 v1, v0, 16, 1
-; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v0
-; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
-; VI-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff
+; VI-NEXT: s_lshl_b32 s0, s0, 16
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_and_b32 s1, s2, 0xffff0000
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_sub_f32_e32 v2, s1, v2
+; VI-NEXT: v_bfe_u32 v3, v2, 16, 1
+; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v2
+; VI-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; VI-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; VI-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_fabs_fadd_bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s3, s2, 0x7fff
@@ -72,14 +69,13 @@ define amdgpu_kernel void @fneg_fabs_fadd_bf16(ptr addrspace(1) %out, bfloat %x,
;
; GFX11-LABEL: fneg_fabs_fadd_bf16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s1, s0, 0x7fff
-; GFX11-NEXT: s_and_b32 s0, s0, 0xffff0000
-; GFX11-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_sub_f32_e64 v0, s0, s1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_and_b32 s3, s2, 0x7fff
+; GFX11-NEXT: s_and_b32 s2, s2, 0xffff0000
+; GFX11-NEXT: s_lshl_b32 s3, s3, 16
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_sub_f32_e64 v0, s2, s3
; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -88,7 +84,6 @@ define amdgpu_kernel void @fneg_fabs_fadd_bf16(ptr addrspace(1) %out, bfloat %x,
; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v2
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_store_d16_hi_b16 v3, v0, s[0:1]
; GFX11-NEXT: s_endpgm
%fabs = call bfloat @llvm.fabs.bf16(bfloat %x)
@@ -101,51 +96,48 @@ define amdgpu_kernel void @fneg_fabs_fadd_bf16(ptr addrspace(1) %out, bfloat %x,
define amdgpu_kernel void @fneg_fabs_fmul_bf16(ptr addrspace(1) %out, bfloat %x, bfloat %y) {
; CI-LABEL: fneg_fabs_fmul_bf16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshl_b32 s3, s2, 16
-; CI-NEXT: s_and_b32 s2, s2, 0xffff0000
-; CI-NEXT: v_mov_b32_e32 v0, s3
-; CI-NEXT: v_mul_f32_e64 v0, s2, -|v0|
-; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_lshl_b32 s0, s2, 16
; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: s_and_b32 s1, s2, 0xffff0000
+; CI-NEXT: v_mov_b32_e32 v2, s0
+; CI-NEXT: v_mul_f32_e64 v2, s1, -|v2|
+; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; CI-NEXT: flat_store_short v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: fneg_fabs_fmul_bf16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_or_b32 s3, s2, 0x8000
-; VI-NEXT: s_lshl_b32 s3, s3, 16
-; VI-NEXT: s_and_b32 s2, s2, 0xffff0000
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mul_f32_e32 v0, s2, v0
-; VI-NEXT: v_bfe_u32 v1, v0, 16, 1
-; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v0
-; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
-; VI-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_or_b32 s0, s2, 0x8000
+; VI-NEXT: s_lshl_b32 s0, s0, 16
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_and_b32 s1, s2, 0xffff0000
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_mul_f32_e32 v2, s1, v2
+; VI-NEXT: v_bfe_u32 v3, v2, 16, 1
+; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v2
+; VI-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; VI-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; VI-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_fabs_fmul_bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_or_b32 s3, s2, 0x8000
@@ -164,14 +156,13 @@ define amdgpu_kernel void @fneg_fabs_fmul_bf16(ptr addrspace(1) %out, bfloat %x,
;
; GFX11-LABEL: fneg_fabs_fmul_bf16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_or_b32 s1, s0, 0x8000
-; GFX11-NEXT: s_and_b32 s0, s0, 0xffff0000
-; GFX11-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mul_f32_e64 v0, s0, s1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_or_b32 s3, s2, 0x8000
+; GFX11-NEXT: s_and_b32 s2, s2, 0xffff0000
+; GFX11-NEXT: s_lshl_b32 s3, s3, 16
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: v_mul_f32_e64 v0, s2, s3
; GFX11-NEXT: v_bfe_u32 v1, v0, 16, 1
; GFX11-NEXT: v_or_b32_e32 v2, 0x400000, v0
; GFX11-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
@@ -180,7 +171,6 @@ define amdgpu_kernel void @fneg_fabs_fmul_bf16(ptr addrspace(1) %out, bfloat %x,
; GFX11-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_cndmask_b32 v0, v1, v2
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_store_d16_hi_b16 v3, v0, s[0:1]
; GFX11-NEXT: s_endpgm
%fabs = call bfloat @llvm.fabs.bf16(bfloat %x)
@@ -194,40 +184,23 @@ define amdgpu_kernel void @fneg_fabs_fmul_bf16(ptr addrspace(1) %out, bfloat %x,
; (fabs (bf16 bitcast (i16 a))) => (bf16 bitcast (and (i16 a), 0x7FFFFFFF))
; unless isFabsFree returns true
define amdgpu_kernel void @fneg_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {
-; CI-LABEL: fneg_fabs_free_bf16:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_bitset1_b32 s2, 15
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_short v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: fneg_fabs_free_bf16:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset1_b32 s2, 15
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: flat_store_short v[0:1], v2
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: fneg_fabs_free_bf16:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_or_b32 s0, s2, 0x8000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_short v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_fabs_free_bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bitset1_b32 s2, 15
@@ -237,9 +210,7 @@ define amdgpu_kernel void @fneg_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {
;
; GFX11-TRUE16-LABEL: fneg_fabs_free_bf16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_bitset1_b32 s2, 15
@@ -250,9 +221,7 @@ define amdgpu_kernel void @fneg_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {
;
; GFX11-FAKE16-LABEL: fneg_fabs_free_bf16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_bitset1_b32 s2, 15
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -267,40 +236,23 @@ define amdgpu_kernel void @fneg_fabs_free_bf16(ptr addrspace(1) %out, i16 %in) {
}
define amdgpu_kernel void @fneg_fabs_bf16(ptr addrspace(1) %out, bfloat %in) {
-; CI-LABEL: fneg_fabs_bf16:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_bitset1_b32 s2, 15
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_short v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: fneg_fabs_bf16:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset1_b32 s2, 15
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: flat_store_short v[0:1], v2
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: fneg_fabs_bf16:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_or_b32 s0, s2, 0x8000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_short v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_fabs_bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bitset1_b32 s2, 15
@@ -310,9 +262,7 @@ define amdgpu_kernel void @fneg_fabs_bf16(ptr addrspace(1) %out, bfloat %in) {
;
; GFX11-TRUE16-LABEL: fneg_fabs_bf16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_bitset1_b32 s2, 15
@@ -323,9 +273,7 @@ define amdgpu_kernel void @fneg_fabs_bf16(ptr addrspace(1) %out, bfloat %in) {
;
; GFX11-FAKE16-LABEL: fneg_fabs_bf16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_bitset1_b32 s2, 15
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -398,70 +346,67 @@ define amdgpu_kernel void @v_fneg_fabs_bf16(ptr addrspace(1) %out, ptr addrspace
define amdgpu_kernel void @s_fneg_fabs_v2bf16_non_bc_src(ptr addrspace(1) %out, <2 x bfloat> %in) {
; CI-LABEL: s_fneg_fabs_v2bf16_non_bc_src:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshl_b32 s3, s2, 16
-; CI-NEXT: s_and_b32 s2, s2, 0xffff0000
-; CI-NEXT: v_add_f32_e64 v1, s2, 2.0
-; CI-NEXT: v_add_f32_e64 v0, s3, 1.0
-; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; CI-NEXT: v_or_b32_e32 v2, 0x80008000, v0
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_lshl_b32 s0, s2, 16
+; CI-NEXT: s_and_b32 s2, s2, 0xffff0000
+; CI-NEXT: v_add_f32_e64 v2, s2, 2.0
+; CI-NEXT: v_add_f32_e64 v1, s0, 1.0
+; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; CI-NEXT: v_lshr_b64 v[2:3], v[1:2], 16
; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: v_or_b32_e32 v2, 0x80008000, v2
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_fneg_fabs_v2bf16_non_bc_src:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b32 s3, s2, 16
-; VI-NEXT: v_add_f32_e64 v0, s3, 1.0
-; VI-NEXT: v_bfe_u32 v1, v0, 16, 1
-; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v0
-; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
-; VI-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; VI-NEXT: s_and_b32 s2, s2, 0xffff0000
-; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; VI-NEXT: v_add_f32_e64 v1, s2, 2.0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_lshl_b32 s0, s2, 16
+; VI-NEXT: v_add_f32_e64 v1, s0, 1.0
; VI-NEXT: v_bfe_u32 v2, v1, 16, 1
; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
; VI-NEXT: v_or_b32_e32 v3, 0x400000, v1
; VI-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; VI-NEXT: s_and_b32 s0, s2, 0xffff0000
; VI-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
-; VI-NEXT: v_or_b32_e32 v2, 0x80008000, v0
-; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_add_f32_e64 v2, s0, 2.0
+; VI-NEXT: v_bfe_u32 v3, v2, 16, 1
+; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v2
+; VI-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; VI-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; VI-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: v_lshrrev_b64 v[2:3], 16, v[1:2]
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_or_b32_e32 v2, 0x80008000, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fneg_fabs_v2bf16_non_bc_src:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s3, s2, 0xffff0000
; GFX9-NEXT: v_add_f32_e64 v1, s3, 2.0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s2, s2, 16
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_add_f32_e64 v2, s2, 1.0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
@@ -480,13 +425,12 @@ define amdgpu_kernel void @s_fneg_fabs_v2bf16_non_bc_src(ptr addrspace(1) %out,
;
; GFX11-TRUE16-LABEL: s_fneg_fabs_v2bf16_non_bc_src:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s0, 0xffff0000
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, s1, 2.0
-; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, s0, 1.0
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s2, 0xffff0000
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 16
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v0, s3, 2.0
+; GFX11-TRUE16-NEXT: v_add_f32_e64 v1, s2, 1.0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
@@ -509,19 +453,17 @@ define amdgpu_kernel void @s_fneg_fabs_v2bf16_non_bc_src(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v1, 0x80008000, v1
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: s_fneg_fabs_v2bf16_non_bc_src:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s0, 16
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0xffff0000
-; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, s1, 1.0
-; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, s0, 2.0
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s2, 16
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 0xffff0000
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v0, s3, 1.0
+; GFX11-FAKE16-NEXT: v_add_f32_e64 v1, s2, 2.0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1
@@ -545,7 +487,6 @@ define amdgpu_kernel void @s_fneg_fabs_v2bf16_non_bc_src(ptr addrspace(1) %out,
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, 0x80008000, v0
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
%add = fadd <2 x bfloat> %in, <bfloat 1.0, bfloat 2.0>
@@ -558,40 +499,23 @@ define amdgpu_kernel void @s_fneg_fabs_v2bf16_non_bc_src(ptr addrspace(1) %out,
; FIXME: single bit op
; Combine turns this into integer op when bitcast source (from load)
define amdgpu_kernel void @s_fneg_fabs_v2bf16_bc_src(ptr addrspace(1) %out, <2 x bfloat> %in) {
-; CI-LABEL: s_fneg_fabs_v2bf16_bc_src:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_or_b32 s2, s2, 0x80008000
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_dword v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: s_fneg_fabs_v2bf16_bc_src:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_or_b32 s2, s2, 0x80008000
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: s_fneg_fabs_v2bf16_bc_src:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_or_b32 s0, s2, 0x80008000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_dword v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fneg_fabs_v2bf16_bc_src:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_or_b32 s2, s2, 0x80008000
@@ -601,9 +525,7 @@ define amdgpu_kernel void @s_fneg_fabs_v2bf16_bc_src(ptr addrspace(1) %out, <2 x
;
; GFX11-LABEL: s_fneg_fabs_v2bf16_bc_src:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_or_b32 s2, s2, 0x80008000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -624,13 +546,13 @@ define amdgpu_kernel void @fneg_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat>
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_or_b32 s3, s3, 0x80008000
-; CI-NEXT: s_or_b32 s2, s2, 0x80008000
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v0, s2
-; CI-NEXT: v_mov_b32_e32 v1, s3
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: s_or_b32 s0, s3, 0x80008000
+; CI-NEXT: s_or_b32 s1, s2, 0x80008000
+; CI-NEXT: v_mov_b32_e32 v2, s1
+; CI-NEXT: v_mov_b32_e32 v3, s0
+; CI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CI-NEXT: s_endpgm
;
; VI-LABEL: fneg_fabs_v4bf16:
@@ -640,23 +562,23 @@ define amdgpu_kernel void @fneg_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat>
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s4, s2, 16
-; VI-NEXT: s_lshr_b32 s5, s3, 16
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_lshr_b32 s0, s2, 16
+; VI-NEXT: s_lshr_b32 s1, s3, 16
; VI-NEXT: s_bitset1_b32 s3, 15
+; VI-NEXT: s_bitset1_b32 s1, 15
; VI-NEXT: s_bitset1_b32 s2, 15
-; VI-NEXT: s_bitset1_b32 s5, 15
-; VI-NEXT: s_bitset1_b32 s4, 15
+; VI-NEXT: s_bitset1_b32 s0, 15
; VI-NEXT: s_and_b32 s3, 0xffff, s3
-; VI-NEXT: s_lshl_b32 s5, s5, 16
+; VI-NEXT: s_lshl_b32 s1, s1, 16
; VI-NEXT: s_and_b32 s2, 0xffff, s2
-; VI-NEXT: s_lshl_b32 s4, s4, 16
-; VI-NEXT: s_or_b32 s3, s3, s5
-; VI-NEXT: s_or_b32 s2, s2, s4
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: s_lshl_b32 s0, s0, 16
+; VI-NEXT: s_or_b32 s1, s3, s1
+; VI-NEXT: s_or_b32 s0, s2, s0
; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_fabs_v4bf16:
@@ -703,75 +625,68 @@ define amdgpu_kernel void @fneg_fabs_v4bf16(ptr addrspace(1) %out, <4 x bfloat>
define amdgpu_kernel void @fold_user_fneg_fabs_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) #0 {
; CI-LABEL: fold_user_fneg_fabs_v2bf16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s3, s2, 0x7fff
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0x7fff
; CI-NEXT: s_and_b32 s2, s2, 0x7fff0000
-; CI-NEXT: v_mul_f32_e64 v0, s2, -4.0
-; CI-NEXT: s_lshl_b32 s2, s3, 16
-; CI-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; CI-NEXT: v_mul_f32_e64 v0, s2, -4.0
-; CI-NEXT: v_lshr_b64 v[0:1], v[0:1], 16
-; CI-NEXT: v_mov_b32_e32 v2, s1
-; CI-NEXT: v_mov_b32_e32 v1, s0
-; CI-NEXT: flat_store_dword v[1:2], v0
+; CI-NEXT: v_mul_f32_e64 v1, s2, -4.0
+; CI-NEXT: s_lshl_b32 s0, s0, 16
+; CI-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; CI-NEXT: v_mul_f32_e64 v1, s0, -4.0
+; CI-NEXT: v_lshr_b64 v[2:3], v[1:2], 16
+; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: fold_user_fneg_fabs_v2bf16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s3, s2, 0x7fff
-; VI-NEXT: s_lshl_b32 s3, s3, 16
-; VI-NEXT: v_mul_f32_e64 v0, s3, -4.0
-; VI-NEXT: v_bfe_u32 v1, v0, 16, 1
-; VI-NEXT: v_add_u32_e32 v1, vcc, v1, v0
-; VI-NEXT: s_lshr_b32 s2, s2, 16
-; VI-NEXT: v_add_u32_e32 v1, vcc, 0x7fff, v1
-; VI-NEXT: s_and_b32 s2, s2, 0x7fff
-; VI-NEXT: v_or_b32_e32 v2, 0x400000, v0
-; VI-NEXT: v_cmp_u_f32_e32 vcc, v0, v0
-; VI-NEXT: s_lshl_b32 s2, s2, 16
-; VI-NEXT: v_cndmask_b32_e32 v0, v1, v2, vcc
-; VI-NEXT: v_mul_f32_e64 v1, s2, -4.0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff
+; VI-NEXT: s_lshl_b32 s0, s0, 16
+; VI-NEXT: v_mul_f32_e64 v1, s0, -4.0
; VI-NEXT: v_bfe_u32 v2, v1, 16, 1
; VI-NEXT: v_add_u32_e32 v2, vcc, v2, v1
; VI-NEXT: v_add_u32_e32 v2, vcc, 0x7fff, v2
; VI-NEXT: v_or_b32_e32 v3, 0x400000, v1
; VI-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff0000
; VI-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
-; VI-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; VI-NEXT: v_lshrrev_b64 v[0:1], 16, v[0:1]
-; VI-NEXT: v_mov_b32_e32 v2, s1
-; VI-NEXT: v_mov_b32_e32 v1, s0
-; VI-NEXT: flat_store_dword v[1:2], v0
+; VI-NEXT: v_mul_f32_e64 v2, s0, -4.0
+; VI-NEXT: v_bfe_u32 v3, v2, 16, 1
+; VI-NEXT: v_add_u32_e32 v3, vcc, v3, v2
+; VI-NEXT: v_add_u32_e32 v3, vcc, 0x7fff, v3
+; VI-NEXT: v_or_b32_e32 v4, 0x400000, v2
+; VI-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
+; VI-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
+; VI-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; VI-NEXT: v_lshrrev_b64 v[2:3], 16, v[1:2]
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fold_user_fneg_fabs_v2bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s3, s2, 0x7fff
-; GFX9-NEXT: s_lshl_b32 s3, s3, 16
-; GFX9-NEXT: v_mul_f32_e64 v1, s3, -4.0
+; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff0000
+; GFX9-NEXT: v_mul_f32_e64 v1, s2, -4.0
; GFX9-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX9-NEXT: s_lshr_b32 s2, s2, 16
; GFX9-NEXT: v_add_u32_e32 v2, v2, v1
-; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff
; GFX9-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX9-NEXT: v_add_u32_e32 v2, 0x7fff, v2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v1, v1
-; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_lshl_b32 s2, s3, 16
; GFX9-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc
; GFX9-NEXT: v_mul_f32_e64 v2, s2, -4.0
; GFX9-NEXT: v_bfe_u32 v3, v2, 16, 1
@@ -781,29 +696,27 @@ define amdgpu_kernel void @fold_user_fneg_fabs_v2bf16(ptr addrspace(1) %out, <2
; GFX9-NEXT: v_cmp_u_f32_e32 vcc, v2, v2
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v4, vcc
; GFX9-NEXT: v_mov_b32_e32 v3, 0xffff
-; GFX9-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-NEXT: v_and_b32_sdwa v1, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
-; GFX9-NEXT: v_lshl_or_b32 v1, v2, 16, v1
+; GFX9-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-NEXT: v_and_b32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
+; GFX9-NEXT: v_lshl_or_b32 v1, v1, 16, v2
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: fold_user_fneg_fabs_v2bf16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-TRUE16-NEXT: s_and_b32 s0, s0, 0x7fff
-; GFX11-TRUE16-NEXT: s_and_b32 s1, s1, 0x7fff
-; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-TRUE16-NEXT: v_mul_f32_e64 v1, s0, -4.0
-; GFX11-TRUE16-NEXT: v_mul_f32_e64 v0, s1, -4.0
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_and_b32 s3, s2, 0x7fff0000
+; GFX11-TRUE16-NEXT: s_and_b32 s2, s2, 0x7fff
+; GFX11-TRUE16-NEXT: v_mul_f32_e64 v0, s3, -4.0
+; GFX11-TRUE16-NEXT: s_lshl_b32 s2, s2, 16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mul_f32_e64 v1, s2, -4.0
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -819,44 +732,39 @@ define amdgpu_kernel void @fold_user_fneg_fabs_v2bf16(ptr addrspace(1) %out, <2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_store_b32 v2, v1, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: fold_user_fneg_fabs_v2bf16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: s_and_b32 s1, s0, 0x7fff
-; GFX11-FAKE16-NEXT: s_lshr_b32 s0, s0, 16
-; GFX11-FAKE16-NEXT: s_lshl_b32 s1, s1, 16
-; GFX11-FAKE16-NEXT: s_and_b32 s0, s0, 0x7fff
-; GFX11-FAKE16-NEXT: v_mul_f32_e64 v0, s1, -4.0
-; GFX11-FAKE16-NEXT: s_lshl_b32 s0, s0, 16
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_mul_f32_e64 v1, s0, -4.0
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_and_b32 s3, s2, 0x7fff
+; GFX11-FAKE16-NEXT: s_and_b32 s2, s2, 0x7fff0000
+; GFX11-FAKE16-NEXT: s_lshl_b32 s3, s3, 16
+; GFX11-FAKE16-NEXT: v_mul_f32_e64 v1, s2, -4.0
+; GFX11-FAKE16-NEXT: v_mul_f32_e64 v0, s3, -4.0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, v3, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, v2, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v3, v5 :: v_dual_and_b32 v0, 0xffff, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v1, v3, v5 :: v_dual_and_b32 v0, 0xffff, v0
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
%fabs = call <2 x bfloat> @llvm.fabs.v2bf16(<2 x bfloat> %in)
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
index 00d53cd265c28..e1895d07aa8b8 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.f16.ll
@@ -8,18 +8,16 @@
define amdgpu_kernel void @fneg_fabs_fadd_f16(ptr addrspace(1) %out, half %x, half %y) {
; CI-LABEL: fneg_fabs_fadd_f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[8:9], 0x2
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s1, s0, 16
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s1
-; CI-NEXT: v_cvt_f32_f16_e64 v1, |s0|
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_lshr_b32 s3, s2, 16
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s3
+; CI-NEXT: v_cvt_f32_f16_e64 v1, |s2|
; CI-NEXT: v_sub_f32_e32 v0, v0, v1
; CI-NEXT: v_cvt_f16_f32_e32 v2, v0
-; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_short v[0:1], v2
@@ -27,24 +25,22 @@ define amdgpu_kernel void @fneg_fabs_fadd_f16(ptr addrspace(1) %out, half %x, ha
;
; VI-LABEL: fneg_fabs_fadd_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s3, s2, 16
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_sub_f16_e64 v2, s3, |v0|
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_lshr_b32 s0, s2, 16
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_sub_f16_e64 v2, s0, |v2|
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_fabs_fadd_f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s3, s2, 16
@@ -55,9 +51,7 @@ define amdgpu_kernel void @fneg_fabs_fadd_f16(ptr addrspace(1) %out, half %x, ha
;
; GFX11-TRUE16-LABEL: fneg_fabs_fadd_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -68,9 +62,7 @@ define amdgpu_kernel void @fneg_fabs_fadd_f16(ptr addrspace(1) %out, half %x, ha
;
; GFX11-FAKE16-LABEL: fneg_fabs_fadd_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -88,18 +80,16 @@ define amdgpu_kernel void @fneg_fabs_fadd_f16(ptr addrspace(1) %out, half %x, ha
define amdgpu_kernel void @fneg_fabs_fmul_f16(ptr addrspace(1) %out, half %x, half %y) {
; CI-LABEL: fneg_fabs_fmul_f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[8:9], 0x2
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s1, s0, 16
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s1
-; CI-NEXT: v_cvt_f32_f16_e64 v1, -|s0|
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_lshr_b32 s3, s2, 16
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s3
+; CI-NEXT: v_cvt_f32_f16_e64 v1, -|s2|
; CI-NEXT: v_mul_f32_e32 v0, v0, v1
; CI-NEXT: v_cvt_f16_f32_e32 v2, v0
-; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_short v[0:1], v2
@@ -107,24 +97,22 @@ define amdgpu_kernel void @fneg_fabs_fmul_f16(ptr addrspace(1) %out, half %x, ha
;
; VI-LABEL: fneg_fabs_fmul_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s3, s2, 16
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mul_f16_e64 v2, s3, -|v0|
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_lshr_b32 s0, s2, 16
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mul_f16_e64 v2, s0, -|v2|
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_fabs_fmul_f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshr_b32 s3, s2, 16
@@ -135,9 +123,7 @@ define amdgpu_kernel void @fneg_fabs_fmul_f16(ptr addrspace(1) %out, half %x, ha
;
; GFX11-TRUE16-LABEL: fneg_fabs_fmul_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -148,9 +134,7 @@ define amdgpu_kernel void @fneg_fabs_fmul_f16(ptr addrspace(1) %out, half %x, ha
;
; GFX11-FAKE16-LABEL: fneg_fabs_fmul_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -169,40 +153,23 @@ define amdgpu_kernel void @fneg_fabs_fmul_f16(ptr addrspace(1) %out, half %x, ha
; (fabs (f16 bitcast (i16 a))) => (f16 bitcast (and (i16 a), 0x7FFFFFFF))
; unless isFabsFree returns true
define amdgpu_kernel void @fneg_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
-; CI-LABEL: fneg_fabs_free_f16:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_bitset1_b32 s2, 15
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_short v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: fneg_fabs_free_f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset1_b32 s2, 15
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: flat_store_short v[0:1], v2
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: fneg_fabs_free_f16:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_or_b32 s0, s2, 0x8000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_short v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_fabs_free_f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bitset1_b32 s2, 15
@@ -212,9 +179,7 @@ define amdgpu_kernel void @fneg_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
;
; GFX11-TRUE16-LABEL: fneg_fabs_free_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_bitset1_b32 s2, 15
@@ -225,9 +190,7 @@ define amdgpu_kernel void @fneg_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
;
; GFX11-FAKE16-LABEL: fneg_fabs_free_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_bitset1_b32 s2, 15
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -242,40 +205,23 @@ define amdgpu_kernel void @fneg_fabs_free_f16(ptr addrspace(1) %out, i16 %in) {
}
define amdgpu_kernel void @fneg_fabs_f16(ptr addrspace(1) %out, half %in) {
-; CI-LABEL: fneg_fabs_f16:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_bitset1_b32 s2, 15
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_short v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: fneg_fabs_f16:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset1_b32 s2, 15
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: flat_store_short v[0:1], v2
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: fneg_fabs_f16:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_or_b32 s0, s2, 0x8000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_short v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_fabs_f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bitset1_b32 s2, 15
@@ -285,9 +231,7 @@ define amdgpu_kernel void @fneg_fabs_f16(ptr addrspace(1) %out, half %in) {
;
; GFX11-TRUE16-LABEL: fneg_fabs_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_bitset1_b32 s2, 15
@@ -298,9 +242,7 @@ define amdgpu_kernel void @fneg_fabs_f16(ptr addrspace(1) %out, half %in) {
;
; GFX11-FAKE16-LABEL: fneg_fabs_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_bitset1_b32 s2, 15
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -373,52 +315,48 @@ define amdgpu_kernel void @v_fneg_fabs_f16(ptr addrspace(1) %out, ptr addrspace(
define amdgpu_kernel void @s_fneg_fabs_v2f16_non_bc_src(ptr addrspace(1) %out, <2 x half> %in) {
; CI-LABEL: s_fneg_fabs_v2f16_non_bc_src:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[8:9], 0x2
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s1, s0, 16
-; CI-NEXT: v_cvt_f32_f16_e32 v1, s1
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_add_f32_e32 v1, 2.0, v1
-; CI-NEXT: v_add_f32_e32 v0, 1.0, v0
-; CI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_or_b32_e32 v0, v0, v1
-; CI-NEXT: v_or_b32_e32 v2, 0x80008000, v0
-; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_lshr_b32 s3, s2, 16
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s3
+; CI-NEXT: v_cvt_f32_f16_e32 v1, s2
+; CI-NEXT: v_add_f32_e32 v0, 2.0, v0
+; CI-NEXT: v_cvt_f16_f32_e32 v2, v0
+; CI-NEXT: v_add_f32_e32 v0, 1.0, v1
+; CI-NEXT: v_cvt_f16_f32_e32 v3, v0
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: v_or_b32_e32 v2, v3, v2
+; CI-NEXT: v_or_b32_e32 v2, 0x80008000, v2
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_fneg_fabs_v2f16_non_bc_src:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: v_mov_b32_e32 v0, 0x4000
+; VI-NEXT: v_mov_b32_e32 v2, 0x4000
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s3, s2, 16
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: v_add_f16_e64 v1, s2, 1.0
-; VI-NEXT: v_add_f16_sdwa v0, v2, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v0, v1, v0
-; VI-NEXT: v_or_b32_e32 v2, 0x80008000, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT: s_lshr_b32 s0, s2, 16
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: v_add_f16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: v_add_f16_e64 v3, s2, 1.0
+; VI-NEXT: v_or_b32_e32 v2, v3, v2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_or_b32_e32 v2, 0x80008000, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fneg_fabs_v2f16_non_bc_src:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 0x40003c00
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -429,9 +367,7 @@ define amdgpu_kernel void @s_fneg_fabs_v2f16_non_bc_src(ptr addrspace(1) %out, <
;
; GFX11-LABEL: s_fneg_fabs_v2f16_non_bc_src:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_pk_add_f16 v0, 0x40003c00, s2
@@ -449,40 +385,23 @@ define amdgpu_kernel void @s_fneg_fabs_v2f16_non_bc_src(ptr addrspace(1) %out, <
; FIXME: single bit op
; Combine turns this into integer op when bitcast source (from load)
define amdgpu_kernel void @s_fneg_fabs_v2f16_bc_src(ptr addrspace(1) %out, <2 x half> %in) {
-; CI-LABEL: s_fneg_fabs_v2f16_bc_src:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_or_b32 s2, s2, 0x80008000
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_dword v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: s_fneg_fabs_v2f16_bc_src:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_or_b32 s2, s2, 0x80008000
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: s_fneg_fabs_v2f16_bc_src:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_or_b32 s0, s2, 0x80008000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_dword v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fneg_fabs_v2f16_bc_src:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_or_b32 s2, s2, 0x80008000
@@ -492,9 +411,7 @@ define amdgpu_kernel void @s_fneg_fabs_v2f16_bc_src(ptr addrspace(1) %out, <2 x
;
; GFX11-LABEL: s_fneg_fabs_v2f16_bc_src:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_or_b32 s2, s2, 0x80008000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -515,13 +432,13 @@ define amdgpu_kernel void @fneg_fabs_v4f16(ptr addrspace(1) %out, <4 x half> %in
; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CIVI-NEXT: s_waitcnt lgkmcnt(0)
-; CIVI-NEXT: s_or_b32 s3, s3, 0x80008000
-; CIVI-NEXT: s_or_b32 s2, s2, 0x80008000
-; CIVI-NEXT: v_mov_b32_e32 v3, s1
-; CIVI-NEXT: v_mov_b32_e32 v0, s2
-; CIVI-NEXT: v_mov_b32_e32 v1, s3
-; CIVI-NEXT: v_mov_b32_e32 v2, s0
-; CIVI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: s_or_b32 s0, s3, 0x80008000
+; CIVI-NEXT: s_or_b32 s1, s2, 0x80008000
+; CIVI-NEXT: v_mov_b32_e32 v2, s1
+; CIVI-NEXT: v_mov_b32_e32 v3, s0
+; CIVI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_fabs_v4f16:
@@ -556,50 +473,47 @@ define amdgpu_kernel void @fneg_fabs_v4f16(ptr addrspace(1) %out, <4 x half> %in
define amdgpu_kernel void @fold_user_fneg_fabs_v2f16(ptr addrspace(1) %out, <2 x half> %in) #0 {
; CI-LABEL: fold_user_fneg_fabs_v2f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[8:9], 0x2
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s1, s0, 16
-; CI-NEXT: v_cvt_f32_f16_e64 v1, |s1|
-; CI-NEXT: v_cvt_f32_f16_e64 v0, |s0|
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_mul_f32_e32 v1, -4.0, v1
+; CI-NEXT: s_lshr_b32 s3, s2, 16
+; CI-NEXT: v_cvt_f32_f16_e64 v0, |s3|
+; CI-NEXT: v_cvt_f32_f16_e64 v1, |s2|
; CI-NEXT: v_mul_f32_e32 v0, -4.0, v0
-; CI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_or_b32_e32 v2, v0, v1
-; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_cvt_f16_f32_e32 v2, v0
+; CI-NEXT: v_mul_f32_e32 v0, -4.0, v1
+; CI-NEXT: v_cvt_f16_f32_e32 v3, v0
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: v_or_b32_e32 v2, v3, v2
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: fold_user_fneg_fabs_v2f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: v_mov_b32_e32 v0, 0xc400
+; VI-NEXT: v_mov_b32_e32 v2, 0xc400
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s3, s2, 16
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: v_mul_f16_e64 v1, |s2|, -4.0
-; VI-NEXT: v_mul_f16_sdwa v0, |v2|, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v2, v1, v0
-; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_bfe_u32 s1, s2, 0xf0010
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0x7fff7fff
+; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_mul_f16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: v_mul_f16_e64 v3, s0, -4.0
+; VI-NEXT: v_or_b32_e32 v2, v3, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fold_user_fneg_fabs_v2f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0x7fff7fff
@@ -609,9 +523,7 @@ define amdgpu_kernel void @fold_user_fneg_fabs_v2f16(ptr addrspace(1) %out, <2 x
;
; GFX11-LABEL: fold_user_fneg_fabs_v2f16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, s2, 0x7fff7fff
diff --git a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
index 3b0edfde1249d..a3897466e4dba 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg-fabs.ll
@@ -20,10 +20,10 @@ define amdgpu_kernel void @fneg_fabsf_fadd_f32(ptr addrspace(1) %out, float %x,
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_sub_f32_e64 v2, s3, |v0|
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_sub_f32_e64 v2, s3, |v2|
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%fabs = call float @llvm.fabs.f32(float %x)
@@ -51,10 +51,10 @@ define amdgpu_kernel void @fneg_fabsf_fmul_f32(ptr addrspace(1) %out, float %x,
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mul_f32_e64 v2, s3, -|v0|
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mul_f32_e64 v2, s3, -|v2|
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%fabs = call float @llvm.fabs.f32(float %x)
@@ -67,10 +67,9 @@ define amdgpu_kernel void @fneg_fabsf_fmul_f32(ptr addrspace(1) %out, float %x,
define amdgpu_kernel void @fneg_fabsf_free_f32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: fneg_fabsf_free_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_or_b32 s4, s2, 0x80000000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -79,13 +78,12 @@ define amdgpu_kernel void @fneg_fabsf_free_f32(ptr addrspace(1) %out, i32 %in) {
;
; VI-LABEL: fneg_fabsf_free_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset1_b32 s2, 31
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_or_b32 s0, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%bc = bitcast i32 %in to float
@@ -98,10 +96,9 @@ define amdgpu_kernel void @fneg_fabsf_free_f32(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @fneg_fabsf_f32(ptr addrspace(1) %out, float %in) {
; SI-LABEL: fneg_fabsf_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_or_b32 s4, s2, 0x80000000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -110,13 +107,12 @@ define amdgpu_kernel void @fneg_fabsf_f32(ptr addrspace(1) %out, float %in) {
;
; VI-LABEL: fneg_fabsf_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset1_b32 s2, 31
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_or_b32 s0, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%fabs = call float @llvm.fabs.f32(float %in)
@@ -168,28 +164,29 @@ define amdgpu_kernel void @fneg_fabsf_v2f32(ptr addrspace(1) %out, <2 x float> %
; SI-LABEL: fneg_fabsf_v2f32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_bitset1_b32 s5, 31
-; SI-NEXT: s_bitset1_b32 s4, 31
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_bitset1_b32 s3, 31
+; SI-NEXT: s_bitset1_b32 s2, 31
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s3
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fneg_fabsf_v2f32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitset1_b32 s3, 31
-; VI-NEXT: s_bitset1_b32 s2, 31
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_or_b32 s0, s3, 0x80000000
+; VI-NEXT: s_or_b32 s1, s2, 0x80000000
+; VI-NEXT: v_mov_b32_e32 v2, s1
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
%fabs = call <2 x float> @llvm.fabs.v2f32(<2 x float> %in)
%fsub = fsub <2 x float> <float -0.000000e+00, float -0.000000e+00>, %fabs
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll b/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
index 63aadaacbeb3a..4407c8df2737e 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
@@ -6,40 +6,23 @@
; RUN: llc -amdgpu-scalarize-global-loads=false -mcpu=gfx1100 -mattr=-real-true16 -mtriple=amdgcn--amdhsa < %s | FileCheck -enable-var-scope -check-prefixes=GFX11,GFX11-FAKE16 %s
define amdgpu_kernel void @s_fneg_bf16(ptr addrspace(1) %out, bfloat %in) #0 {
-; CI-LABEL: s_fneg_bf16:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_xor_b32 s2, s2, 0x8000
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_short v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; GFX8-LABEL: s_fneg_bf16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX8-NEXT: s_add_i32 s12, s12, s17
-; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: flat_store_short v[0:1], v2
-; GFX8-NEXT: s_endpgm
+; CIVI-LABEL: s_fneg_bf16:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_xor_b32 s0, s2, 0x8000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_short v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fneg_bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_xor_b32 s2, s2, 0x8000
@@ -49,9 +32,7 @@ define amdgpu_kernel void @s_fneg_bf16(ptr addrspace(1) %out, bfloat %in) #0 {
;
; GFX11-TRUE16-LABEL: s_fneg_bf16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_xor_b32 s2, s2, 0x8000
@@ -62,9 +43,7 @@ define amdgpu_kernel void @s_fneg_bf16(ptr addrspace(1) %out, bfloat %in) #0 {
;
; GFX11-FAKE16-LABEL: s_fneg_bf16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_xor_b32 s2, s2, 0x8000
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -159,40 +138,23 @@ define amdgpu_kernel void @v_fneg_bf16(ptr addrspace(1) %out, ptr addrspace(1) %
}
define amdgpu_kernel void @s_fneg_free_bf16(ptr addrspace(1) %out, i16 %in) #0 {
-; CI-LABEL: s_fneg_free_bf16:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_xor_b32 s2, s2, 0x8000
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_short v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; GFX8-LABEL: s_fneg_free_bf16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX8-NEXT: s_add_i32 s12, s12, s17
-; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: flat_store_short v[0:1], v2
-; GFX8-NEXT: s_endpgm
+; CIVI-LABEL: s_fneg_free_bf16:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_xor_b32 s0, s2, 0x8000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_short v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fneg_free_bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_xor_b32 s2, s2, 0x8000
@@ -202,9 +164,7 @@ define amdgpu_kernel void @s_fneg_free_bf16(ptr addrspace(1) %out, i16 %in) #0 {
;
; GFX11-TRUE16-LABEL: s_fneg_free_bf16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_xor_b32 s2, s2, 0x8000
@@ -215,9 +175,7 @@ define amdgpu_kernel void @s_fneg_free_bf16(ptr addrspace(1) %out, i16 %in) #0 {
;
; GFX11-FAKE16-LABEL: s_fneg_free_bf16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_xor_b32 s2, s2, 0x8000
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -354,42 +312,39 @@ define amdgpu_kernel void @v_fneg_fold_bf16(ptr addrspace(1) %out, ptr addrspace
define amdgpu_kernel void @s_fneg_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) #0 {
; CI-LABEL: s_fneg_v2bf16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s3, s2, 0xffff0000
-; CI-NEXT: s_xor_b32 s2, s2, 0x8000
-; CI-NEXT: s_and_b32 s2, s2, 0xffff
-; CI-NEXT: s_or_b32 s2, s2, s3
-; CI-NEXT: s_add_i32 s2, s2, 0x80000000
-; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: s_xor_b32 s1, s2, 0x8000
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0xffff0000
+; CI-NEXT: s_and_b32 s1, s1, 0xffff
+; CI-NEXT: s_or_b32 s0, s1, s0
+; CI-NEXT: s_add_i32 s0, s0, 0x80000000
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; GFX8-LABEL: s_fneg_v2bf16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX8-NEXT: s_add_i32 s12, s12, s17
; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x80008000
; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_xor_b32 s0, s2, 0x80008000
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: s_fneg_v2bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_xor_b32 s2, s2, 0x80008000
@@ -399,9 +354,7 @@ define amdgpu_kernel void @s_fneg_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in
;
; GFX11-LABEL: s_fneg_v2bf16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_xor_b32 s2, s2, 0x80008000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -557,42 +510,39 @@ define amdgpu_kernel void @v_fneg_v2bf16(ptr addrspace(1) %out, ptr addrspace(1)
define amdgpu_kernel void @fneg_free_v2bf16(ptr addrspace(1) %out, i32 %in) #0 {
; CI-LABEL: fneg_free_v2bf16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_and_b32 s3, s2, 0xffff0000
-; CI-NEXT: s_xor_b32 s2, s2, 0x8000
-; CI-NEXT: s_and_b32 s2, s2, 0xffff
-; CI-NEXT: s_or_b32 s2, s2, s3
-; CI-NEXT: s_add_i32 s2, s2, 0x80000000
-; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: s_xor_b32 s1, s2, 0x8000
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_and_b32 s0, s2, 0xffff0000
+; CI-NEXT: s_and_b32 s1, s1, 0xffff
+; CI-NEXT: s_or_b32 s0, s1, s0
+; CI-NEXT: s_add_i32 s0, s0, 0x80000000
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; GFX8-LABEL: fneg_free_v2bf16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX8-NEXT: s_add_i32 s12, s12, s17
; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x80008000
; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_xor_b32 s0, s2, 0x80008000
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_free_v2bf16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_xor_b32 s2, s2, 0x80008000
@@ -602,9 +552,7 @@ define amdgpu_kernel void @fneg_free_v2bf16(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX11-LABEL: fneg_free_v2bf16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_xor_b32 s2, s2, 0x80008000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -1013,5 +961,3 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CIVI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.f16.ll b/llvm/test/CodeGen/AMDGPU/fneg.f16.ll
index 3140b87c8108e..b397022b749f4 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.f16.ll
@@ -7,40 +7,23 @@
; FIXME: Should be able to do scalar op
define amdgpu_kernel void @s_fneg_f16(ptr addrspace(1) %out, half %in) #0 {
-; CI-LABEL: s_fneg_f16:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_xor_b32 s2, s2, 0x8000
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_short v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; GFX8-LABEL: s_fneg_f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX8-NEXT: s_add_i32 s12, s12, s17
-; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: flat_store_short v[0:1], v2
-; GFX8-NEXT: s_endpgm
+; CIVI-LABEL: s_fneg_f16:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_xor_b32 s0, s2, 0x8000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_short v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fneg_f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_xor_b32 s2, s2, 0x8000
@@ -50,9 +33,7 @@ define amdgpu_kernel void @s_fneg_f16(ptr addrspace(1) %out, half %in) #0 {
;
; GFX11-TRUE16-LABEL: s_fneg_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_xor_b32 s2, s2, 0x8000
@@ -63,9 +44,7 @@ define amdgpu_kernel void @s_fneg_f16(ptr addrspace(1) %out, half %in) #0 {
;
; GFX11-FAKE16-LABEL: s_fneg_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_xor_b32 s2, s2, 0x8000
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -160,40 +139,23 @@ define amdgpu_kernel void @v_fneg_f16(ptr addrspace(1) %out, ptr addrspace(1) %i
}
define amdgpu_kernel void @s_fneg_free_f16(ptr addrspace(1) %out, i16 %in) #0 {
-; CI-LABEL: s_fneg_free_f16:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_xor_b32 s2, s2, 0x8000
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_short v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; GFX8-LABEL: s_fneg_free_f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX8-NEXT: s_add_i32 s12, s12, s17
-; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x8000
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: flat_store_short v[0:1], v2
-; GFX8-NEXT: s_endpgm
+; CIVI-LABEL: s_fneg_free_f16:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_xor_b32 s0, s2, 0x8000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_short v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fneg_free_f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_xor_b32 s2, s2, 0x8000
@@ -203,9 +165,7 @@ define amdgpu_kernel void @s_fneg_free_f16(ptr addrspace(1) %out, i16 %in) #0 {
;
; GFX11-TRUE16-LABEL: s_fneg_free_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_xor_b32 s2, s2, 0x8000
@@ -216,9 +176,7 @@ define amdgpu_kernel void @s_fneg_free_f16(ptr addrspace(1) %out, i16 %in) #0 {
;
; GFX11-FAKE16-LABEL: s_fneg_free_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_xor_b32 s2, s2, 0x8000
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -309,40 +267,23 @@ define amdgpu_kernel void @v_fneg_fold_f16(ptr addrspace(1) %out, ptr addrspace(
}
define amdgpu_kernel void @s_fneg_v2f16(ptr addrspace(1) %out, <2 x half> %in) #0 {
-; CI-LABEL: s_fneg_v2f16:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_xor_b32 s2, s2, 0x80008000
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_dword v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; GFX8-LABEL: s_fneg_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX8-NEXT: s_add_i32 s12, s12, s17
-; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x80008000
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: flat_store_dword v[0:1], v2
-; GFX8-NEXT: s_endpgm
+; CIVI-LABEL: s_fneg_v2f16:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_xor_b32 s0, s2, 0x80008000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_dword v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: s_fneg_v2f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_xor_b32 s2, s2, 0x80008000
@@ -352,9 +293,7 @@ define amdgpu_kernel void @s_fneg_v2f16(ptr addrspace(1) %out, <2 x half> %in) #
;
; GFX11-LABEL: s_fneg_v2f16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_xor_b32 s2, s2, 0x80008000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -484,40 +423,23 @@ define amdgpu_kernel void @v_fneg_v2f16(ptr addrspace(1) %out, ptr addrspace(1)
}
define amdgpu_kernel void @fneg_free_v2f16(ptr addrspace(1) %out, i32 %in) #0 {
-; CI-LABEL: fneg_free_v2f16:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_xor_b32 s2, s2, 0x80008000
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_dword v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; GFX8-LABEL: fneg_free_v2f16:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX8-NEXT: s_add_i32 s12, s12, s17
-; GFX8-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GFX8-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_xor_b32 s2, s2, 0x80008000
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
-; GFX8-NEXT: flat_store_dword v[0:1], v2
-; GFX8-NEXT: s_endpgm
+; CIVI-LABEL: fneg_free_v2f16:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: s_xor_b32 s0, s2, 0x80008000
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: flat_store_dword v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX9-LABEL: fneg_free_v2f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_xor_b32 s2, s2, 0x80008000
@@ -527,9 +449,7 @@ define amdgpu_kernel void @fneg_free_v2f16(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX11-LABEL: fneg_free_v2f16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_xor_b32 s2, s2, 0x80008000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.ll b/llvm/test/CodeGen/AMDGPU/fneg.ll
index 4fe0882a19f18..ab10044a1b9f7 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.ll
@@ -10,33 +10,31 @@
define amdgpu_kernel void @s_fneg_f32(ptr addrspace(1) %out, float %in) {
; SI-LABEL: s_fneg_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_xor_b32 s4, s6, 0x80000000
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_xor_b32 s0, s2, 0x80000000
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_fneg_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_xor_b32 s2, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_xor_b32 s0, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_fneg_f32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_xor_b32 s2, s2, 0x80000000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -52,28 +50,29 @@ define amdgpu_kernel void @s_fneg_v2f32(ptr addrspace(1) nocapture %out, <2 x fl
; SI-LABEL: s_fneg_v2f32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_xor_b32 s5, s5, 0x80000000
-; SI-NEXT: s_xor_b32 s4, s4, 0x80000000
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_xor_b32 s0, s3, 0x80000000
+; SI-NEXT: s_xor_b32 s1, s2, 0x80000000
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: v_mov_b32_e32 v1, s0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_fneg_v2f32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_xor_b32 s3, s3, 0x80000000
-; VI-NEXT: s_xor_b32 s2, s2, 0x80000000
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_xor_b32 s0, s3, 0x80000000
+; VI-NEXT: s_xor_b32 s1, s2, 0x80000000
+; VI-NEXT: v_mov_b32_e32 v2, s1
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_fneg_v2f32:
@@ -153,31 +152,29 @@ define amdgpu_kernel void @s_fneg_v4f32(ptr addrspace(1) nocapture %out, <4 x fl
define amdgpu_kernel void @fsub0_f32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: fsub0_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_sub_f32_e64 v0, 0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_sub_f32_e64 v0, 0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fsub0_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_sub_f32_e64 v2, 0, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_sub_f32_e64 v2, 0, s2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: fsub0_f32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_sub_f32_e64 v1, 0, s2
@@ -192,33 +189,31 @@ define amdgpu_kernel void @fsub0_f32(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @fneg_free_f32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: fneg_free_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_xor_b32 s4, s6, 0x80000000
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_xor_b32 s0, s2, 0x80000000
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fneg_free_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_xor_b32 s2, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_xor_b32 s0, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: fneg_free_f32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_xor_b32 s2, s2, 0x80000000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -234,31 +229,28 @@ define amdgpu_kernel void @fneg_free_f32(ptr addrspace(1) %out, i32 %in) {
define amdgpu_kernel void @fneg_fold_f32(ptr addrspace(1) %out, float %in) {
; SI-LABEL: fneg_fold_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, -s6, s6
+; SI-NEXT: v_mul_f32_e64 v0, -s4, s4
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fneg_fold_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mul_f32_e64 v2, -s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mul_f32_e64 v2, -s2, s2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: fneg_fold_f32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mul_f32_e64 v1, -s2, s2
@@ -274,31 +266,29 @@ define amdgpu_kernel void @fneg_fold_f32(ptr addrspace(1) %out, float %in) {
define amdgpu_kernel void @bitpreserve_fneg_f32(ptr addrspace(1) %out, float %in) {
; SI-LABEL: bitpreserve_fneg_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mul_f32_e64 v0, s6, -4.0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mul_f32_e64 v0, s2, -4.0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: bitpreserve_fneg_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mul_f32_e64 v2, s2, -4.0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mul_f32_e64 v2, s2, -4.0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: bitpreserve_fneg_f32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mul_f32_e64 v1, s2, -4.0
@@ -315,33 +305,31 @@ define amdgpu_kernel void @bitpreserve_fneg_f32(ptr addrspace(1) %out, float %in
define amdgpu_kernel void @s_fneg_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_fneg_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_xor_b32 s4, s6, 0x80000000
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_xor_b32 s0, s2, 0x80000000
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_fneg_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_xor_b32 s2, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_xor_b32 s0, s2, 0x80000000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_fneg_i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_xor_b32 s2, s2, 0x80000000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -366,31 +354,29 @@ define i32 @v_fneg_i32(i32 %in) {
define amdgpu_kernel void @s_fneg_i32_fp_use(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_fneg_i32_fp_use:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_sub_f32_e64 v0, 2.0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_sub_f32_e64 v0, 2.0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_fneg_i32_fp_use:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_sub_f32_e64 v2, 2.0, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_sub_f32_e64 v2, 2.0, s2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_fneg_i32_fp_use:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_sub_f32_e64 v1, 2.0, s2
@@ -548,34 +534,29 @@ define i16 @v_fneg_i16(i16 %in) {
define amdgpu_kernel void @s_fneg_i16_fp_use(ptr addrspace(1) %out, i16 %in) {
; SI-LABEL: s_fneg_i16_fp_use:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_sub_f32_e32 v0, 2.0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_fneg_i16_fp_use:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_sub_f16_e64 v2, 2.0, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_sub_f16_e64 v2, 2.0, s2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: s_fneg_i16_fp_use:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_sub_f16_e64 v0.l, 2.0, s2
@@ -584,9 +565,7 @@ define amdgpu_kernel void @s_fneg_i16_fp_use(ptr addrspace(1) %out, i16 %in) {
;
; GFX11-FAKE16-LABEL: s_fneg_i16_fp_use:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_sub_f16_e64 v1, 2.0, s2
@@ -634,33 +613,31 @@ define half @v_fneg_i16_fp_use(i16 %in) {
define amdgpu_kernel void @s_fneg_v2i16(ptr addrspace(1) %out, i32 %arg) {
; SI-LABEL: s_fneg_v2i16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_xor_b32 s4, s6, 0x80008000
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_xor_b32 s0, s2, 0x80008000
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_fneg_v2i16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_xor_b32 s2, s2, 0x80008000
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_xor_b32 s0, s2, 0x80008000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_fneg_v2i16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_xor_b32 s2, s2, 0x80008000
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -686,45 +663,40 @@ define <2 x i16> @v_fneg_v2i16(<2 x i16> %in) {
define amdgpu_kernel void @s_fneg_v2i16_fp_use(ptr addrspace(1) %out, i32 %arg) {
; SI-LABEL: s_fneg_v2i16_fp_use:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_lshr_b32 s3, s2, 16
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s3
+; SI-NEXT: v_cvt_f32_f16_e32 v1, s2
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s1, s0, 16
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s1
-; SI-NEXT: v_cvt_f32_f16_e32 v1, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_sub_f32_e32 v0, 2.0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
; SI-NEXT: v_sub_f32_e32 v1, 2.0, v1
; SI-NEXT: v_cvt_f16_f32_e32 v1, v1
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_fneg_v2i16_fp_use:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x4000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x4000
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s3, s2, 16
-; VI-NEXT: v_mov_b32_e32 v2, s3
-; VI-NEXT: v_sub_f16_e64 v1, 2.0, s2
-; VI-NEXT: v_sub_f16_sdwa v0, v0, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v2, v1, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_lshr_b32 s0, s2, 16
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: v_sub_f16_sdwa v2, v2, v3 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: v_sub_f16_e64 v3, 2.0, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_or_b32_e32 v2, v3, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: s_fneg_v2i16_fp_use:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_pk_add_f16 v1, s2, 2.0 op_sel_hi:[1,0] neg_lo:[1,0] neg_hi:[1,0]
diff --git a/llvm/test/CodeGen/AMDGPU/fp-classify.ll b/llvm/test/CodeGen/AMDGPU/fp-classify.ll
index 303864ff9434a..4965606540f28 100644
--- a/llvm/test/CodeGen/AMDGPU/fp-classify.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp-classify.ll
@@ -10,35 +10,33 @@ declare double @llvm.fabs.f64(double) #1
define amdgpu_kernel void @test_isinf_pattern(ptr addrspace(1) nocapture %out, float %x) #0 {
; SI-LABEL: test_isinf_pattern:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: v_mov_b32_e32 v0, 0x204
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_class_f32_e32 vcc, s4, v0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_isinf_pattern:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x204
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x204
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
+; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_isinf_pattern:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f32_e64 s2, s2, 0x204
@@ -56,35 +54,33 @@ define amdgpu_kernel void @test_isinf_pattern(ptr addrspace(1) nocapture %out, f
define amdgpu_kernel void @test_not_isinf_pattern_0(ptr addrspace(1) nocapture %out, float %x) #0 {
; SI-LABEL: test_not_isinf_pattern_0:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: v_mov_b32_e32 v0, 0x7f800000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_nlg_f32_e64 s[4:5], |s4|, v0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_nlg_f32_e64 s[0:1], |s2|, v0
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_not_isinf_pattern_0:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x7f800000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x7f800000
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_nlg_f32_e64 s[2:3], |s2|, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cmp_nlg_f32_e64 s[0:1], |s2|, v2
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_not_isinf_pattern_0:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_nlg_f32_e64 s2, 0x7f800000, |s2|
@@ -137,35 +133,33 @@ define amdgpu_kernel void @test_not_isinf_pattern_1(ptr addrspace(1) nocapture %
define amdgpu_kernel void @test_isfinite_pattern_0(ptr addrspace(1) nocapture %out, float %x) #0 {
; SI-LABEL: test_isfinite_pattern_0:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: v_mov_b32_e32 v0, 0x1f8
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_class_f32_e32 vcc, s4, v0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_isfinite_pattern_0:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
+; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_isfinite_pattern_0:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f32_e64 s2, s2, 0x1f8
@@ -185,35 +179,33 @@ define amdgpu_kernel void @test_isfinite_pattern_0(ptr addrspace(1) nocapture %o
define amdgpu_kernel void @test_isfinite_pattern_1(ptr addrspace(1) nocapture %out, float %x) #0 {
; SI-LABEL: test_isfinite_pattern_1:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: v_mov_b32_e32 v0, 0x1f8
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_class_f32_e32 vcc, s4, v0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_isfinite_pattern_1:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
+; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_isfinite_pattern_1:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f32_e64 s2, s2, 0x1f8
@@ -232,33 +224,30 @@ define amdgpu_kernel void @test_isfinite_pattern_1(ptr addrspace(1) nocapture %o
define amdgpu_kernel void @test_isfinite_not_pattern_0(ptr addrspace(1) nocapture %out, float %x) #0 {
; SI-LABEL: test_isfinite_not_pattern_0:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_o_f32_e64 s[4:5], s6, s6
+; SI-NEXT: v_cmp_o_f32_e64 s[4:5], s4, s4
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_isfinite_not_pattern_0:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_o_f32_e64 s[2:3], s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cmp_o_f32_e64 s[0:1], s2, s2
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_isfinite_not_pattern_0:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_o_f32_e64 s2, s2, s2
@@ -279,39 +268,37 @@ define amdgpu_kernel void @test_isfinite_not_pattern_0(ptr addrspace(1) nocaptur
define amdgpu_kernel void @test_isfinite_not_pattern_1(ptr addrspace(1) nocapture %out, float %x) #0 {
; SI-LABEL: test_isfinite_not_pattern_1:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: v_mov_b32_e32 v0, 0x7f800000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_o_f32_e64 s[4:5], s6, s6
-; SI-NEXT: v_cmp_neq_f32_e32 vcc, s6, v0
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], vcc
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_o_f32_e64 s[0:1], s2, s2
+; SI-NEXT: v_cmp_neq_f32_e32 vcc, s2, v0
+; SI-NEXT: s_and_b64 s[0:1], s[0:1], vcc
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_isfinite_not_pattern_1:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x7f800000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x7f800000
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_o_f32_e64 s[2:3], s6, s6
-; VI-NEXT: v_cmp_neq_f32_e32 vcc, s6, v0
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], vcc
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cmp_o_f32_e64 s[0:1], s2, s2
+; VI-NEXT: v_cmp_neq_f32_e32 vcc, s2, v2
+; VI-NEXT: s_and_b64 s[0:1], s[0:1], vcc
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_isfinite_not_pattern_1:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_o_f32_e64 s3, s2, s2
@@ -350,14 +337,14 @@ define amdgpu_kernel void @test_isfinite_not_pattern_2(ptr addrspace(1) nocaptur
; VI-LABEL: test_isfinite_not_pattern_2:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x7f800000
+; VI-NEXT: v_mov_b32_e32 v2, 0x7f800000
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_o_f32_e64 s[4:5], s2, s2
-; VI-NEXT: v_cmp_neq_f32_e64 s[2:3], |s3|, v0
-; VI-NEXT: s_and_b64 s[2:3], s[4:5], s[2:3]
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cmp_o_f32_e64 s[0:1], s2, s2
+; VI-NEXT: v_cmp_neq_f32_e64 s[2:3], |s3|, v2
+; VI-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -386,39 +373,37 @@ define amdgpu_kernel void @test_isfinite_not_pattern_2(ptr addrspace(1) nocaptur
define amdgpu_kernel void @test_isfinite_not_pattern_3(ptr addrspace(1) nocapture %out, float %x) #0 {
; SI-LABEL: test_isfinite_not_pattern_3:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: v_mov_b32_e32 v0, 0x7f800000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_u_f32_e64 s[4:5], s6, s6
-; SI-NEXT: v_cmp_neq_f32_e64 s[6:7], |s6|, v0
-; SI-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_u_f32_e64 s[0:1], s2, s2
+; SI-NEXT: v_cmp_neq_f32_e64 s[2:3], |s2|, v0
+; SI-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_isfinite_not_pattern_3:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x7f800000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x7f800000
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_u_f32_e64 s[2:3], s6, s6
-; VI-NEXT: v_cmp_neq_f32_e64 s[4:5], |s6|, v0
-; VI-NEXT: s_and_b64 s[2:3], s[2:3], s[4:5]
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[2:3]
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cmp_u_f32_e64 s[0:1], s2, s2
+; VI-NEXT: v_cmp_neq_f32_e64 s[2:3], |s2|, v2
+; VI-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_isfinite_not_pattern_3:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_u_f32_e64 s3, s2, s2
@@ -440,35 +425,33 @@ define amdgpu_kernel void @test_isfinite_not_pattern_3(ptr addrspace(1) nocaptur
define amdgpu_kernel void @test_isfinite_pattern_4(ptr addrspace(1) nocapture %out, float %x) #0 {
; SI-LABEL: test_isfinite_pattern_4:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: v_mov_b32_e32 v0, 0x1f8
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_class_f32_e32 vcc, s4, v0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_isfinite_pattern_4:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
+; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_isfinite_pattern_4:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f32_e64 s2, s2, 0x1f8
@@ -488,35 +471,33 @@ define amdgpu_kernel void @test_isfinite_pattern_4(ptr addrspace(1) nocapture %o
define amdgpu_kernel void @test_isfinite_pattern_4_commute_and(ptr addrspace(1) nocapture %out, float %x) #0 {
; SI-LABEL: test_isfinite_pattern_4_commute_and:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: v_mov_b32_e32 v0, 0x1f8
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_class_f32_e32 vcc, s4, v0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_isfinite_pattern_4_commute_and:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
+; VI-NEXT: v_cmp_class_f32_e32 vcc, s2, v2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_isfinite_pattern_4_commute_and:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f32_e64 s2, s2, 0x1f8
@@ -536,44 +517,45 @@ define amdgpu_kernel void @test_isfinite_pattern_4_commute_and(ptr addrspace(1)
define amdgpu_kernel void @test_not_isfinite_pattern_4_wrong_ord_test(ptr addrspace(1) nocapture %out, float %x, [8 x i32], float %y) #0 {
; SI-LABEL: test_not_isfinite_pattern_4_wrong_ord_test:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x9
-; SI-NEXT: s_load_dword s0, s[4:5], 0x14
-; SI-NEXT: s_load_dword s1, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s11, 0xf000
-; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_load_dword s3, s[4:5], 0x14
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: v_mov_b32_e32 v0, 0x1f8
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s0
-; SI-NEXT: v_cmp_o_f32_e32 vcc, s1, v1
-; SI-NEXT: v_cmp_class_f32_e64 s[0:1], s1, v0
+; SI-NEXT: v_mov_b32_e32 v1, s3
+; SI-NEXT: v_cmp_o_f32_e32 vcc, s2, v1
+; SI-NEXT: v_cmp_class_f32_e64 s[0:1], s2, v0
; SI-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_not_isfinite_pattern_4_wrong_ord_test:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s0, s[4:5], 0x50
-; VI-NEXT: s_load_dword s1, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s0
-; VI-NEXT: v_cmp_class_f32_e32 vcc, s1, v0
-; VI-NEXT: v_cmp_o_f32_e64 s[0:1], s1, v1
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], vcc
-; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: s_load_dword s3, s[4:5], 0x50
+; VI-NEXT: v_mov_b32_e32 v2, 0x1f8
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cmp_o_f32_e32 vcc, s2, v3
+; VI-NEXT: v_cmp_class_f32_e64 s[0:1], s2, v2
+; VI-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
-; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_not_isfinite_pattern_4_wrong_ord_test:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x50
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_o_f32_e64 s3, s2, s3
@@ -595,36 +577,34 @@ define amdgpu_kernel void @test_not_isfinite_pattern_4_wrong_ord_test(ptr addrsp
define amdgpu_kernel void @test_isinf_pattern_f16(ptr addrspace(1) nocapture %out, half %x) #0 {
; SI-LABEL: test_isinf_pattern_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b32 s5, 0x7f800000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v0, |s4|
-; SI-NEXT: v_cmp_eq_f32_e32 vcc, s5, v0
+; SI-NEXT: s_mov_b32 s3, 0x7f800000
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_f32_f16_e64 v0, |s2|
+; SI-NEXT: v_cmp_eq_f32_e32 vcc, s3, v0
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_isinf_pattern_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x204
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x204
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_class_f16_e32 vcc, s2, v0
+; VI-NEXT: v_cmp_class_f16_e32 vcc, s2, v2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_isinf_pattern_f16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f16_e64 s2, s2, 0x204
@@ -642,39 +622,37 @@ define amdgpu_kernel void @test_isinf_pattern_f16(ptr addrspace(1) nocapture %ou
define amdgpu_kernel void @test_isfinite_pattern_0_f16(ptr addrspace(1) nocapture %out, half %x) #0 {
; SI-LABEL: test_isfinite_pattern_0_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x9
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s11, 0xf000
-; SI-NEXT: s_mov_b32 s10, -1
-; SI-NEXT: s_mov_b32 s1, 0x7f800000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: v_cvt_f32_f16_e64 v1, |s0|
+; SI-NEXT: s_mov_b32 s3, 0x7f800000
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
+; SI-NEXT: v_cvt_f32_f16_e64 v1, |s2|
; SI-NEXT: v_cmp_o_f32_e32 vcc, v0, v0
-; SI-NEXT: v_cmp_neq_f32_e64 s[0:1], s1, v1
+; SI-NEXT: v_cmp_neq_f32_e64 s[0:1], s3, v1
; SI-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_isfinite_pattern_0_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_class_f16_e32 vcc, s2, v0
+; VI-NEXT: v_cmp_class_f16_e32 vcc, s2, v2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_isfinite_pattern_0_f16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f16_e64 s2, s2, 0x1f8
@@ -694,36 +672,34 @@ define amdgpu_kernel void @test_isfinite_pattern_0_f16(ptr addrspace(1) nocaptur
define amdgpu_kernel void @test_isfinite_pattern_4_f16(ptr addrspace(1) nocapture %out, half %x) #0 {
; SI-LABEL: test_isfinite_pattern_4_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b32 s5, 0x7f800000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e64 v0, |s4|
-; SI-NEXT: v_cmp_lg_f32_e32 vcc, s5, v0
+; SI-NEXT: s_mov_b32 s3, 0x7f800000
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_f32_f16_e64 v0, |s2|
+; SI-NEXT: v_cmp_lg_f32_e32 vcc, s3, v0
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_isfinite_pattern_4_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v0, 0x1f8
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v2, 0x1f8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_class_f16_e32 vcc, s2, v0
+; VI-NEXT: v_cmp_class_f16_e32 vcc, s2, v2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-LABEL: test_isfinite_pattern_4_f16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_class_f16_e64 s2, s2, 0x1f8
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
index 8f03abd1124f7..7e2a50a670dc4 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-atomics-gfx90a.ll
@@ -20,24 +20,22 @@ declare double @llvm.amdgcn.ds.fadd.f64(ptr addrspace(3) nocapture, double, i32,
define amdgpu_kernel void @raw_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: raw_buffer_atomic_add_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 offen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_buffer_atomic_add_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 offen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_buffer_atomic_add_noret_f64:
@@ -47,7 +45,7 @@ define amdgpu_kernel void @raw_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, doub
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -87,32 +85,30 @@ main_body:
define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 4 offen glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 4 offen sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_buffer_atomic_add_rtn_f64_off4_slc:
@@ -123,7 +119,7 @@ define amdgpu_kernel void @raw_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsr
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -142,24 +138,22 @@ main_body:
define amdgpu_kernel void @raw_ptr_buffer_atomic_add_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: raw_ptr_buffer_atomic_add_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 offen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_ptr_buffer_atomic_add_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 offen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_noret_f64:
@@ -169,7 +163,7 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_noret_f64(ptr addrspace(8)
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -209,32 +203,30 @@ main_body:
define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 4 offen glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 4 offen sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_add_rtn_f64_off4_slc:
@@ -245,7 +237,7 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrsp
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -264,24 +256,22 @@ main_body:
define amdgpu_kernel void @struct_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: struct_buffer_atomic_add_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_buffer_atomic_add_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_buffer_atomic_add_noret_f64:
@@ -291,7 +281,7 @@ define amdgpu_kernel void @struct_buffer_atomic_add_noret_f64(<4 x i32> %rsrc, d
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen
; GFX1250-NEXT: s_endpgm
@@ -331,32 +321,30 @@ main_body:
define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_buffer_atomic_add_rtn_f64_off4_slc:
@@ -366,7 +354,7 @@ define amdgpu_kernel void @struct_buffer_atomic_add_rtn_f64_off4_slc(<4 x i32> %
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -385,24 +373,22 @@ main_body:
define amdgpu_kernel void @struct_ptr_buffer_atomic_add_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: struct_ptr_buffer_atomic_add_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_ptr_buffer_atomic_add_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_noret_f64:
@@ -412,7 +398,7 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_noret_f64(ptr addrspace(
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen
; GFX1250-NEXT: s_endpgm
@@ -452,32 +438,30 @@ main_body:
define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_add_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_add_rtn_f64_off4_slc:
@@ -487,7 +471,7 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_add_rtn_f64_off4_slc(ptr add
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_add_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -506,24 +490,22 @@ main_body:
define amdgpu_kernel void @raw_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: raw_buffer_atomic_min_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 offen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_buffer_atomic_min_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 offen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_buffer_atomic_min_noret_f64:
@@ -533,7 +515,7 @@ define amdgpu_kernel void @raw_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, doub
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -573,32 +555,30 @@ main_body:
define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 4 offen glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 4 offen sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_buffer_atomic_min_rtn_f64_off4_slc:
@@ -609,7 +589,7 @@ define amdgpu_kernel void @raw_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsr
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -628,24 +608,22 @@ main_body:
define amdgpu_kernel void @raw_ptr_buffer_atomic_min_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: raw_ptr_buffer_atomic_min_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 offen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_ptr_buffer_atomic_min_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 offen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_min_noret_f64:
@@ -655,7 +633,7 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_noret_f64(ptr addrspace(8)
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -695,32 +673,30 @@ main_body:
define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 4 offen glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 4 offen sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_min_rtn_f64_off4_slc:
@@ -731,7 +707,7 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrsp
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -750,24 +726,22 @@ main_body:
define amdgpu_kernel void @struct_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: struct_buffer_atomic_min_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_buffer_atomic_min_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_buffer_atomic_min_noret_f64:
@@ -777,7 +751,7 @@ define amdgpu_kernel void @struct_buffer_atomic_min_noret_f64(<4 x i32> %rsrc, d
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen
; GFX1250-NEXT: s_endpgm
@@ -817,32 +791,30 @@ main_body:
define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_buffer_atomic_min_rtn_f64_off4_slc:
@@ -852,7 +824,7 @@ define amdgpu_kernel void @struct_buffer_atomic_min_rtn_f64_off4_slc(<4 x i32> %
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -871,24 +843,22 @@ main_body:
define amdgpu_kernel void @struct_ptr_buffer_atomic_min_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: struct_ptr_buffer_atomic_min_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_ptr_buffer_atomic_min_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_min_noret_f64:
@@ -898,7 +868,7 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_noret_f64(ptr addrspace(
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen
; GFX1250-NEXT: s_endpgm
@@ -938,32 +908,30 @@ main_body:
define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_min_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_min_rtn_f64_off4_slc:
@@ -973,7 +941,7 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_min_rtn_f64_off4_slc(ptr add
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_min_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -992,24 +960,22 @@ main_body:
define amdgpu_kernel void @raw_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: raw_buffer_atomic_max_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 offen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_buffer_atomic_max_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 offen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_buffer_atomic_max_noret_f64:
@@ -1019,7 +985,7 @@ define amdgpu_kernel void @raw_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, doub
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -1059,32 +1025,30 @@ main_body:
define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 4 offen glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 4 offen sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:
@@ -1095,7 +1059,7 @@ define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsr
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -1114,24 +1078,22 @@ main_body:
define amdgpu_kernel void @raw_ptr_buffer_atomic_max_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: raw_ptr_buffer_atomic_max_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 offen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_ptr_buffer_atomic_max_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 offen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 offen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_max_noret_f64:
@@ -1141,7 +1103,7 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_noret_f64(ptr addrspace(8)
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null offen
; GFX1250-NEXT: s_endpgm
@@ -1181,32 +1143,30 @@ main_body:
define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 4 offen glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 4 offen sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 4 offen sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:
@@ -1217,7 +1177,7 @@ define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrsp
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_mov_b32 s6, 4
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], s6 offen th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -1236,24 +1196,22 @@ main_body:
define amdgpu_kernel void @struct_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: struct_buffer_atomic_max_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_buffer_atomic_max_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_buffer_atomic_max_noret_f64:
@@ -1263,7 +1221,7 @@ define amdgpu_kernel void @struct_buffer_atomic_max_noret_f64(<4 x i32> %rsrc, d
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen
; GFX1250-NEXT: s_endpgm
@@ -1303,32 +1261,30 @@ main_body:
define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_buffer_atomic_max_rtn_f64_off4_slc:
@@ -1338,7 +1294,7 @@ define amdgpu_kernel void @struct_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> %
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
@@ -1357,24 +1313,22 @@ main_body:
define amdgpu_kernel void @struct_ptr_buffer_atomic_max_noret_f64(ptr addrspace(8) %rsrc, double %data, i32 %vindex) {
; GFX90A-LABEL: struct_ptr_buffer_atomic_max_noret_f64:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s8
-; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_ptr_buffer_atomic_max_noret_f64:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s8
-; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 idxen
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_max_noret_f64:
@@ -1384,7 +1338,7 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_noret_f64(ptr addrspace(
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen
; GFX1250-NEXT: s_endpgm
@@ -1424,32 +1378,30 @@ main_body:
define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrspace(8) %rsrc, double %data, i32 %vindex, ptr addrspace(1) %out) {
; GFX90A-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; GFX90A: ; %bb.0: ; %main_body
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX90A-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX90A-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX90A-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_pk_mov_b32 v[2:3], s[6:7], s[6:7] op_sel:[0,1]
-; GFX90A-NEXT: v_mov_b32_e32 v0, s10
-; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 glc slc
+; GFX90A-NEXT: v_mov_b32_e32 v2, s12
+; GFX90A-NEXT: v_mov_b32_e32 v3, s13
+; GFX90A-NEXT: v_mov_b32_e32 v0, s14
+; GFX90A-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 glc slc
; GFX90A-NEXT: v_mov_b32_e32 v0, 0
; GFX90A-NEXT: s_waitcnt vmcnt(0)
-; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX90A-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX90A-NEXT: s_endpgm
;
; GFX942-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; GFX942: ; %bb.0: ; %main_body
-; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX942-NEXT: s_load_dword s10, s[4:5], 0x3c
-; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX942-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x44
+; GFX942-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
+; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x44
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
-; GFX942-NEXT: v_mov_b32_e32 v0, s10
-; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[0:3], 0 idxen offset:4 sc0 nt
+; GFX942-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-NEXT: v_mov_b32_e32 v0, s14
+; GFX942-NEXT: buffer_atomic_max_f64 v[2:3], v0, s[8:11], 0 idxen offset:4 sc0 nt
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[8:9]
+; GFX942-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; GFX942-NEXT: s_endpgm
;
; GFX1250-LABEL: struct_ptr_buffer_atomic_max_rtn_f64_off4_slc:
@@ -1459,7 +1411,7 @@ define amdgpu_kernel void @struct_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr add
; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x34 nv
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_mov_b64_e32 v[0:1], s[8:9]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
; GFX1250-NEXT: v_mov_b32_e32 v2, s10
; GFX1250-NEXT: buffer_atomic_max_num_f64 v[0:1], v2, s[0:3], null idxen offset:4 th:TH_ATOMIC_NT_RETURN
; GFX1250-NEXT: s_wait_xcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll
index e987190e335d3..8d3cc0880093a 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-atomics.ll
@@ -16,51 +16,41 @@ declare double @llvm.amdgcn.raw.buffer.atomic.fmax.f64(double, <4 x i32>, i32, i
define amdgpu_kernel void @raw_buffer_atomic_min_noret_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {
; SI-LABEL: raw_buffer_atomic_min_noret_f64:
; SI: ; %bb.0: ; %main_body
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xf
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: v_mov_b32_e32 v2, s8
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[0:3], 0 offen
; SI-NEXT: s_endpgm
;
; GFX7-LABEL: raw_buffer_atomic_min_noret_f64:
; GFX7: ; %bb.0: ; %main_body
-; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; GFX7-NEXT: s_load_dword s8, s[4:5], 0xf
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: v_mov_b32_e32 v2, s6
; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[0:3], 0 offen
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: raw_buffer_atomic_min_noret_f64:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s6
-; GFX10-NEXT: v_mov_b32_e32 v1, s7
-; GFX10-NEXT: v_mov_b32_e32 v2, s8
-; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[0:3], 0 offen
+; GFX10-NEXT: v_mov_b32_e32 v0, s12
+; GFX10-NEXT: v_mov_b32_e32 v1, s13
+; GFX10-NEXT: v_mov_b32_e32 v2, s14
+; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[8:11], 0 offen
; GFX10-NEXT: s_endpgm
;
; GFX1030-LABEL: raw_buffer_atomic_min_noret_f64:
; GFX1030: ; %bb.0: ; %main_body
-; GFX1030-NEXT: s_clause 0x2
-; GFX1030-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX1030-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX1030-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1030-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-NEXT: v_mov_b32_e32 v0, s6
-; GFX1030-NEXT: v_mov_b32_e32 v1, s7
-; GFX1030-NEXT: v_mov_b32_e32 v2, s8
+; GFX1030-NEXT: v_mov_b32_e32 v0, s4
+; GFX1030-NEXT: v_mov_b32_e32 v1, s5
+; GFX1030-NEXT: v_mov_b32_e32 v2, s6
; GFX1030-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[0:3], 0 offen
; GFX1030-NEXT: s_endpgm
;
@@ -253,51 +243,41 @@ main_body:
define amdgpu_kernel void @raw_buffer_atomic_max_noret_f64(<4 x i32> inreg %rsrc, double %data, i32 %vindex) {
; SI-LABEL: raw_buffer_atomic_max_noret_f64:
; SI: ; %bb.0: ; %main_body
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xf
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: v_mov_b32_e32 v2, s8
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 0 offen
; SI-NEXT: s_endpgm
;
; GFX7-LABEL: raw_buffer_atomic_max_noret_f64:
; GFX7: ; %bb.0: ; %main_body
-; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; GFX7-NEXT: s_load_dword s8, s[4:5], 0xf
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: v_mov_b32_e32 v2, s6
; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 0 offen
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: raw_buffer_atomic_max_noret_f64:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s6
-; GFX10-NEXT: v_mov_b32_e32 v1, s7
-; GFX10-NEXT: v_mov_b32_e32 v2, s8
-; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 0 offen
+; GFX10-NEXT: v_mov_b32_e32 v0, s12
+; GFX10-NEXT: v_mov_b32_e32 v1, s13
+; GFX10-NEXT: v_mov_b32_e32 v2, s14
+; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[8:11], 0 offen
; GFX10-NEXT: s_endpgm
;
; GFX1030-LABEL: raw_buffer_atomic_max_noret_f64:
; GFX1030: ; %bb.0: ; %main_body
-; GFX1030-NEXT: s_clause 0x2
-; GFX1030-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX1030-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX1030-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1030-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-NEXT: v_mov_b32_e32 v0, s6
-; GFX1030-NEXT: v_mov_b32_e32 v1, s7
-; GFX1030-NEXT: v_mov_b32_e32 v2, s8
+; GFX1030-NEXT: v_mov_b32_e32 v0, s4
+; GFX1030-NEXT: v_mov_b32_e32 v1, s5
+; GFX1030-NEXT: v_mov_b32_e32 v2, s6
; GFX1030-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 0 offen
; GFX1030-NEXT: s_endpgm
;
@@ -424,14 +404,15 @@ main_body:
define amdgpu_kernel void @raw_buffer_atomic_max_rtn_f64_off4_slc(<4 x i32> inreg %rsrc, double %data, i32 %vindex, ptr addrspace(3) %out) {
; SI-LABEL: raw_buffer_atomic_max_rtn_f64_off4_slc:
; SI: ; %bb.0: ; %main_body
-; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9
+; SI-NEXT: s_load_dword s0, s[4:5], 0x10
; SI-NEXT: s_mov_b32 m0, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: v_mov_b32_e32 v2, s6
-; SI-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 4 offen glc slc
-; SI-NEXT: v_mov_b32_e32 v2, s7
+; SI-NEXT: v_mov_b32_e32 v0, s12
+; SI-NEXT: v_mov_b32_e32 v1, s13
+; SI-NEXT: v_mov_b32_e32 v2, s14
+; SI-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[8:11], 4 offen glc slc
+; SI-NEXT: v_mov_b32_e32 v2, s0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: ds_write_b64 v2, v[0:1]
; SI-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
index 2a7027c3393b2..796be04bf94a6 100644
--- a/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp64-min-max-buffer-ptr-atomics.ll
@@ -16,51 +16,41 @@ declare double @llvm.amdgcn.raw.ptr.buffer.atomic.fmax.f64(double, ptr addrspace
define amdgpu_kernel void @raw_ptr_buffer_atomic_min_noret_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {
; SI-LABEL: raw_ptr_buffer_atomic_min_noret_f64:
; SI: ; %bb.0: ; %main_body
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xf
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: v_mov_b32_e32 v2, s8
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[0:3], 0 offen
; SI-NEXT: s_endpgm
;
; GFX7-LABEL: raw_ptr_buffer_atomic_min_noret_f64:
; GFX7: ; %bb.0: ; %main_body
-; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; GFX7-NEXT: s_load_dword s8, s[4:5], 0xf
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: v_mov_b32_e32 v2, s6
; GFX7-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[0:3], 0 offen
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: raw_ptr_buffer_atomic_min_noret_f64:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s6
-; GFX10-NEXT: v_mov_b32_e32 v1, s7
-; GFX10-NEXT: v_mov_b32_e32 v2, s8
-; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[0:3], 0 offen
+; GFX10-NEXT: v_mov_b32_e32 v0, s12
+; GFX10-NEXT: v_mov_b32_e32 v1, s13
+; GFX10-NEXT: v_mov_b32_e32 v2, s14
+; GFX10-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[8:11], 0 offen
; GFX10-NEXT: s_endpgm
;
; GFX1030-LABEL: raw_ptr_buffer_atomic_min_noret_f64:
; GFX1030: ; %bb.0: ; %main_body
-; GFX1030-NEXT: s_clause 0x2
-; GFX1030-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX1030-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX1030-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1030-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-NEXT: v_mov_b32_e32 v0, s6
-; GFX1030-NEXT: v_mov_b32_e32 v1, s7
-; GFX1030-NEXT: v_mov_b32_e32 v2, s8
+; GFX1030-NEXT: v_mov_b32_e32 v0, s4
+; GFX1030-NEXT: v_mov_b32_e32 v1, s5
+; GFX1030-NEXT: v_mov_b32_e32 v2, s6
; GFX1030-NEXT: buffer_atomic_fmin_x2 v[0:1], v2, s[0:3], 0 offen
; GFX1030-NEXT: s_endpgm
;
@@ -253,51 +243,41 @@ main_body:
define amdgpu_kernel void @raw_ptr_buffer_atomic_max_noret_f64(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex) {
; SI-LABEL: raw_ptr_buffer_atomic_max_noret_f64:
; SI: ; %bb.0: ; %main_body
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xf
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: v_mov_b32_e32 v1, s7
-; SI-NEXT: v_mov_b32_e32 v2, s8
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s5
+; SI-NEXT: v_mov_b32_e32 v2, s6
; SI-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 0 offen
; SI-NEXT: s_endpgm
;
; GFX7-LABEL: raw_ptr_buffer_atomic_max_noret_f64:
; GFX7: ; %bb.0: ; %main_body
-; GFX7-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; GFX7-NEXT: s_load_dword s8, s[4:5], 0xf
-; GFX7-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v0, s6
-; GFX7-NEXT: v_mov_b32_e32 v1, s7
-; GFX7-NEXT: v_mov_b32_e32 v2, s8
+; GFX7-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-NEXT: v_mov_b32_e32 v1, s5
+; GFX7-NEXT: v_mov_b32_e32 v2, s6
; GFX7-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 0 offen
; GFX7-NEXT: s_endpgm
;
; GFX10-LABEL: raw_ptr_buffer_atomic_max_noret_f64:
; GFX10: ; %bb.0: ; %main_body
-; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX10-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s6
-; GFX10-NEXT: v_mov_b32_e32 v1, s7
-; GFX10-NEXT: v_mov_b32_e32 v2, s8
-; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 0 offen
+; GFX10-NEXT: v_mov_b32_e32 v0, s12
+; GFX10-NEXT: v_mov_b32_e32 v1, s13
+; GFX10-NEXT: v_mov_b32_e32 v2, s14
+; GFX10-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[8:11], 0 offen
; GFX10-NEXT: s_endpgm
;
; GFX1030-LABEL: raw_ptr_buffer_atomic_max_noret_f64:
; GFX1030: ; %bb.0: ; %main_body
-; GFX1030-NEXT: s_clause 0x2
-; GFX1030-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX1030-NEXT: s_load_dword s8, s[4:5], 0x3c
-; GFX1030-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1030-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; GFX1030-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1030-NEXT: v_mov_b32_e32 v0, s6
-; GFX1030-NEXT: v_mov_b32_e32 v1, s7
-; GFX1030-NEXT: v_mov_b32_e32 v2, s8
+; GFX1030-NEXT: v_mov_b32_e32 v0, s4
+; GFX1030-NEXT: v_mov_b32_e32 v1, s5
+; GFX1030-NEXT: v_mov_b32_e32 v2, s6
; GFX1030-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 0 offen
; GFX1030-NEXT: s_endpgm
;
@@ -424,14 +404,15 @@ main_body:
define amdgpu_kernel void @raw_ptr_buffer_atomic_max_rtn_f64_off4_slc(ptr addrspace(8) inreg %rsrc, double %data, i32 %vindex, ptr addrspace(3) %out) {
; SI-LABEL: raw_ptr_buffer_atomic_max_rtn_f64_off4_slc:
; SI: ; %bb.0: ; %main_body
-; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x9
+; SI-NEXT: s_load_dword s0, s[4:5], 0x10
; SI-NEXT: s_mov_b32 m0, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: v_mov_b32_e32 v2, s6
-; SI-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[0:3], 4 offen glc slc
-; SI-NEXT: v_mov_b32_e32 v2, s7
+; SI-NEXT: v_mov_b32_e32 v0, s12
+; SI-NEXT: v_mov_b32_e32 v1, s13
+; SI-NEXT: v_mov_b32_e32 v2, s14
+; SI-NEXT: buffer_atomic_fmax_x2 v[0:1], v2, s[8:11], 4 offen glc slc
+; SI-NEXT: v_mov_b32_e32 v2, s0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: ds_write_b64 v2, v[0:1]
; SI-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll b/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
index cb0ebc495c5a7..74bdfac6e6991 100644
--- a/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp_to_sint.ll
@@ -10,21 +10,21 @@ declare float @llvm.fabs.f32(float) #1
define amdgpu_kernel void @fp_to_sint_i32(ptr addrspace(1) %out, float %in) {
; SI-LABEL: fp_to_sint_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_i32_f32_e32 v0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_i32_f32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_sint_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_cvt_i32_f32_e32 v0, s2
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -32,9 +32,7 @@ define amdgpu_kernel void @fp_to_sint_i32(ptr addrspace(1) %out, float %in) {
;
; GFX11-SDAG-LABEL: fp_to_sint_i32:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_i32_f32_e32 v1, s2
@@ -71,21 +69,21 @@ define amdgpu_kernel void @fp_to_sint_i32(ptr addrspace(1) %out, float %in) {
define amdgpu_kernel void @fp_to_sint_i32_fabs(ptr addrspace(1) %out, float %in) {
; SI-LABEL: fp_to_sint_i32_fabs:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_i32_f32_e64 v0, |s6|
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_i32_f32_e64 v0, |s2|
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_sint_i32_fabs:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_cvt_i32_f32_e64 v0, |s2|
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -93,9 +91,7 @@ define amdgpu_kernel void @fp_to_sint_i32_fabs(ptr addrspace(1) %out, float %in)
;
; GFX11-SDAG-LABEL: fp_to_sint_i32_fabs:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_i32_f32_e64 v1, |s2|
@@ -134,24 +130,27 @@ define amdgpu_kernel void @fp_to_sint_v2i32(ptr addrspace(1) %out, <2 x float> %
; SI-LABEL: fp_to_sint_v2i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_cvt_i32_f32_e32 v1, s5
-; SI-NEXT: v_cvt_i32_f32_e32 v0, s4
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_i32_f32_e32 v1, s3
+; SI-NEXT: v_cvt_i32_f32_e32 v0, s2
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_sint_v2i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cvt_i32_f32_e32 v1, s3
; VI-NEXT: v_cvt_i32_f32_e32 v0, s2
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_sint_v2i32:
@@ -284,56 +283,57 @@ define amdgpu_kernel void @fp_to_sint_v4i32(ptr addrspace(1) %out, ptr addrspace
define amdgpu_kernel void @fp_to_sint_i64 (ptr addrspace(1) %out, float %in) {
; SI-LABEL: fp_to_sint_i64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b32 s5, 0x2f800000
-; SI-NEXT: s_mov_b32 s6, 0xcf800000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_trunc_f32_e32 v0, s4
-; SI-NEXT: v_mul_f32_e64 v1, |v0|, s5
+; SI-NEXT: s_mov_b32 s3, 0x2f800000
+; SI-NEXT: s_mov_b32 s8, 0xcf800000
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_trunc_f32_e32 v0, s2
+; SI-NEXT: v_mul_f32_e64 v1, |v0|, s3
; SI-NEXT: v_ashrrev_i32_e32 v2, 31, v0
; SI-NEXT: v_floor_f32_e32 v1, v1
; SI-NEXT: v_cvt_u32_f32_e32 v3, v1
-; SI-NEXT: v_fma_f32 v0, v1, s6, |v0|
+; SI-NEXT: v_fma_f32 v0, v1, s8, |v0|
; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: v_xor_b32_e32 v1, v3, v2
; SI-NEXT: v_xor_b32_e32 v0, v0, v2
; SI-NEXT: v_sub_i32_e32 v0, vcc, v0, v2
; SI-NEXT: v_subb_u32_e32 v1, vcc, v1, v2, vcc
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_sint_i64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s4, 0x2f800000
-; VI-NEXT: s_mov_b32 s5, 0xcf800000
-; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0x2f800000
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: v_trunc_f32_e32 v0, s2
-; VI-NEXT: v_mul_f32_e64 v1, |v0|, s4
+; VI-NEXT: v_mul_f32_e64 v1, |v0|, s3
+; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: v_floor_f32_e32 v1, v1
-; VI-NEXT: v_fma_f32 v2, v1, s5, |v0|
+; VI-NEXT: s_mov_b32 s0, 0xcf800000
+; VI-NEXT: v_fma_f32 v2, v1, s0, |v0|
; VI-NEXT: v_cvt_u32_f32_e32 v2, v2
; VI-NEXT: v_cvt_u32_f32_e32 v1, v1
; VI-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_xor_b32_e32 v0, v2, v3
; VI-NEXT: v_xor_b32_e32 v1, v1, v3
; VI-NEXT: v_sub_u32_e32 v0, vcc, v0, v3
; VI-NEXT: v_subb_u32_e32 v1, vcc, v1, v3, vcc
-; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_sint_i64:
; GFX11-SDAG: ; %bb.0: ; %entry
-; GFX11-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_trunc_f32_e32 v0, s0
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_trunc_f32_e32 v0, s2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_mul_f32_e64 v1, 0x2f800000, |v0|
; GFX11-SDAG-NEXT: v_ashrrev_i32_e32 v3, 31, v0
@@ -349,7 +349,6 @@ define amdgpu_kernel void @fp_to_sint_i64 (ptr addrspace(1) %out, float %in) {
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_sub_co_u32 v0, vcc_lo, v0, v3
; GFX11-SDAG-NEXT: v_sub_co_ci_u32_e64 v1, null, v1, v3, vcc_lo
-; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-SDAG-NEXT: s_endpgm
;
@@ -439,25 +438,26 @@ entry:
define amdgpu_kernel void @fp_to_sint_v2i64(ptr addrspace(1) %out, <2 x float> %x) {
; SI-LABEL: fp_to_sint_v2i64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s6, 0x2f800000
-; SI-NEXT: s_mov_b32 s7, 0xcf800000
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
+; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_trunc_f32_e32 v0, s5
-; SI-NEXT: v_trunc_f32_e32 v1, s4
-; SI-NEXT: v_mul_f32_e64 v2, |v0|, s6
+; SI-NEXT: s_mov_b32 s8, 0x2f800000
+; SI-NEXT: s_mov_b32 s9, 0xcf800000
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s0, s4
+; SI-NEXT: s_mov_b32 s1, s5
+; SI-NEXT: v_trunc_f32_e32 v0, s7
+; SI-NEXT: v_trunc_f32_e32 v1, s6
+; SI-NEXT: v_mul_f32_e64 v2, |v0|, s8
; SI-NEXT: v_ashrrev_i32_e32 v3, 31, v0
-; SI-NEXT: v_mul_f32_e64 v4, |v1|, s6
+; SI-NEXT: v_mul_f32_e64 v4, |v1|, s8
; SI-NEXT: v_ashrrev_i32_e32 v5, 31, v1
; SI-NEXT: v_floor_f32_e32 v2, v2
; SI-NEXT: v_floor_f32_e32 v4, v4
; SI-NEXT: v_cvt_u32_f32_e32 v6, v2
-; SI-NEXT: v_fma_f32 v0, v2, s7, |v0|
+; SI-NEXT: v_fma_f32 v0, v2, s9, |v0|
; SI-NEXT: v_cvt_u32_f32_e32 v2, v4
-; SI-NEXT: v_fma_f32 v1, v4, s7, |v1|
+; SI-NEXT: v_fma_f32 v1, v4, s9, |v1|
; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
; SI-NEXT: v_xor_b32_e32 v4, v6, v3
; SI-NEXT: v_cvt_u32_f32_e32 v1, v1
@@ -474,35 +474,36 @@ define amdgpu_kernel void @fp_to_sint_v2i64(ptr addrspace(1) %out, <2 x float> %
; VI-LABEL: fp_to_sint_v2i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s6, 0x2f800000
-; VI-NEXT: s_mov_b32 s7, 0xcf800000
+; VI-NEXT: s_mov_b32 s8, 0x2f800000
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: v_trunc_f32_e32 v0, s5
-; VI-NEXT: v_mul_f32_e64 v1, |v0|, s6
+; VI-NEXT: v_trunc_f32_e32 v0, s3
+; VI-NEXT: v_mul_f32_e64 v1, |v0|, s8
+; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: v_floor_f32_e32 v1, v1
-; VI-NEXT: v_cvt_u32_f32_e32 v2, v1
-; VI-NEXT: v_fma_f32 v1, v1, s7, |v0|
-; VI-NEXT: v_ashrrev_i32_e32 v0, 31, v0
-; VI-NEXT: v_trunc_f32_e32 v4, s4
-; VI-NEXT: v_xor_b32_e32 v3, v2, v0
-; VI-NEXT: v_mul_f32_e64 v2, |v4|, s6
+; VI-NEXT: s_mov_b32 s0, 0xcf800000
+; VI-NEXT: v_fma_f32 v2, v1, s0, |v0|
+; VI-NEXT: v_trunc_f32_e32 v4, s2
+; VI-NEXT: v_cvt_u32_f32_e32 v2, v2
+; VI-NEXT: v_mul_f32_e64 v3, |v4|, s8
; VI-NEXT: v_cvt_u32_f32_e32 v1, v1
-; VI-NEXT: v_floor_f32_e32 v2, v2
-; VI-NEXT: v_cvt_u32_f32_e32 v5, v2
-; VI-NEXT: v_fma_f32 v2, v2, s7, |v4|
-; VI-NEXT: v_cvt_u32_f32_e32 v6, v2
+; VI-NEXT: v_floor_f32_e32 v3, v3
+; VI-NEXT: v_cvt_u32_f32_e32 v5, v3
+; VI-NEXT: v_fma_f32 v3, v3, s0, |v4|
+; VI-NEXT: v_ashrrev_i32_e32 v0, 31, v0
+; VI-NEXT: v_cvt_u32_f32_e32 v6, v3
+; VI-NEXT: v_xor_b32_e32 v2, v2, v0
; VI-NEXT: v_xor_b32_e32 v1, v1, v0
-; VI-NEXT: v_sub_u32_e32 v2, vcc, v1, v0
+; VI-NEXT: v_sub_u32_e32 v2, vcc, v2, v0
+; VI-NEXT: v_subb_u32_e32 v3, vcc, v1, v0, vcc
; VI-NEXT: v_ashrrev_i32_e32 v1, 31, v4
-; VI-NEXT: v_subb_u32_e32 v3, vcc, v3, v0, vcc
; VI-NEXT: v_xor_b32_e32 v0, v6, v1
; VI-NEXT: v_xor_b32_e32 v4, v5, v1
; VI-NEXT: v_sub_u32_e32 v0, vcc, v0, v1
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_subb_u32_e32 v1, vcc, v4, v1, vcc
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_sint_v2i64:
@@ -1091,33 +1092,33 @@ define amdgpu_kernel void @fp_to_sint_v4i64(ptr addrspace(1) %out, <4 x float> %
define amdgpu_kernel void @fp_to_uint_f32_to_i1(ptr addrspace(1) %out, float %in) #0 {
; SI-LABEL: fp_to_uint_f32_to_i1:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_eq_f32_e64 s[4:5], -1.0, s6
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_eq_f32_e64 s[0:1], -1.0, s2
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_uint_f32_to_i1:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_eq_f32_e64 s[4:5], -1.0, s6
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_cmp_eq_f32_e64 s[0:1], -1.0, s2
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_uint_f32_to_i1:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cmp_eq_f32_e64 s2, -1.0, s2
@@ -1169,33 +1170,33 @@ define amdgpu_kernel void @fp_to_uint_f32_to_i1(ptr addrspace(1) %out, float %in
define amdgpu_kernel void @fp_to_uint_fabs_f32_to_i1(ptr addrspace(1) %out, float %in) #0 {
; SI-LABEL: fp_to_uint_fabs_f32_to_i1:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_eq_f32_e64 s[4:5], -1.0, |s6|
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_eq_f32_e64 s[0:1], -1.0, |s2|
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_uint_fabs_f32_to_i1:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_eq_f32_e64 s[4:5], -1.0, |s6|
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_cmp_eq_f32_e64 s[0:1], -1.0, |s2|
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_uint_fabs_f32_to_i1:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cmp_eq_f32_e64 s2, -1.0, |s2|
@@ -1248,21 +1249,21 @@ define amdgpu_kernel void @fp_to_uint_fabs_f32_to_i1(ptr addrspace(1) %out, floa
define amdgpu_kernel void @fp_to_sint_f32_i16(ptr addrspace(1) %out, float %in) #0 {
; SI-LABEL: fp_to_sint_f32_i16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_i32_f32_e32 v0, s6
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_i32_f32_e32 v0, s2
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_sint_f32_i16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_cvt_i32_f32_e32 v0, s2
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: buffer_store_short v0, off, s[0:3], 0
@@ -1270,9 +1271,7 @@ define amdgpu_kernel void @fp_to_sint_f32_i16(ptr addrspace(1) %out, float %in)
;
; GFX11-SDAG-LABEL: fp_to_sint_f32_i16:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_i32_f32_e32 v0, s2
@@ -1323,29 +1322,32 @@ define amdgpu_kernel void @fp_to_sint_v2f32_to_v2i16(ptr addrspace(1) %out, <2 x
; SI-LABEL: fp_to_sint_v2f32_to_v2i16:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_cvt_i32_f32_e32 v0, s5
-; SI-NEXT: v_cvt_i32_f32_e32 v1, s4
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_i32_f32_e32 v0, s3
+; SI-NEXT: v_cvt_i32_f32_e32 v1, s2
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_sint_v2f32_to_v2i16:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s3
; VI-NEXT: v_cvt_i32_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; VI-NEXT: v_cvt_i32_f32_e32 v1, s2
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_sint_v2f32_to_v2i16:
@@ -1399,29 +1401,32 @@ define amdgpu_kernel void @fp_to_sint_f32_to_v2i16(ptr addrspace(1) %out, float
; SI-LABEL: fp_to_sint_f32_to_v2i16:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_cvt_i32_f32_e32 v0, s4
-; SI-NEXT: v_cvt_i32_f32_e32 v1, s5
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_i32_f32_e32 v0, s2
+; SI-NEXT: v_cvt_i32_f32_e32 v1, s3
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_sint_f32_to_v2i16:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: v_cvt_i32_f32_e32 v0, s2
; VI-NEXT: v_cvt_i32_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_sint_f32_to_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll b/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
index 2c5ea86cfe7e7..99158e09a78dc 100644
--- a/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
+++ b/llvm/test/CodeGen/AMDGPU/fp_to_uint.ll
@@ -10,21 +10,21 @@ declare float @llvm.fabs.f32(float) #1
define amdgpu_kernel void @fp_to_uint_f32_to_i32 (ptr addrspace(1) %out, float %in) {
; SI-LABEL: fp_to_uint_f32_to_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_u32_f32_e32 v0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_u32_f32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_uint_f32_to_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_cvt_u32_f32_e32 v0, s2
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -32,9 +32,7 @@ define amdgpu_kernel void @fp_to_uint_f32_to_i32 (ptr addrspace(1) %out, float %
;
; GFX11-SDAG-LABEL: fp_to_uint_f32_to_i32:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v1, s2
@@ -72,24 +70,27 @@ define amdgpu_kernel void @fp_to_uint_v2f32_to_v2i32(ptr addrspace(1) %out, <2 x
; SI-LABEL: fp_to_uint_v2f32_to_v2i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_cvt_u32_f32_e32 v1, s5
-; SI-NEXT: v_cvt_u32_f32_e32 v0, s4
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_u32_f32_e32 v1, s3
+; SI-NEXT: v_cvt_u32_f32_e32 v0, s2
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_uint_v2f32_to_v2i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_cvt_u32_f32_e32 v1, s3
; VI-NEXT: v_cvt_u32_f32_e32 v0, s2
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_uint_v2f32_to_v2i32:
@@ -221,45 +222,44 @@ define amdgpu_kernel void @fp_to_uint_v4f32_to_v4i32(ptr addrspace(1) %out, ptr
define amdgpu_kernel void @fp_to_uint_f32_to_i64(ptr addrspace(1) %out, float %x) {
; SI-LABEL: fp_to_uint_f32_to_i64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b32 s5, 0xcf800000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_trunc_f32_e32 v0, s4
+; SI-NEXT: s_mov_b32 s3, 0xcf800000
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_trunc_f32_e32 v0, s2
; SI-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; SI-NEXT: v_floor_f32_e32 v2, v1
; SI-NEXT: v_cvt_u32_f32_e32 v1, v2
-; SI-NEXT: v_fma_f32 v0, v2, s5, v0
+; SI-NEXT: v_fma_f32 v0, v2, s3, v0
; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_uint_f32_to_i64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xcf800000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s4, 0xcf800000
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_trunc_f32_e32 v0, s2
; VI-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; VI-NEXT: v_floor_f32_e32 v2, v1
-; VI-NEXT: v_fma_f32 v0, v2, s3, v0
+; VI-NEXT: v_fma_f32 v0, v2, s4, v0
; VI-NEXT: v_cvt_u32_f32_e32 v1, v2
; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
-; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_uint_f32_to_i64:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-SDAG-NEXT: v_trunc_f32_e32 v0, s0
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_trunc_f32_e32 v0, s2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; GFX11-SDAG-NEXT: v_floor_f32_e32 v1, v1
@@ -267,7 +267,6 @@ define amdgpu_kernel void @fp_to_uint_f32_to_i64(ptr addrspace(1) %out, float %x
; GFX11-SDAG-NEXT: v_fmamk_f32 v0, v1, 0xcf800000, v0
; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v1, v1
; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-SDAG-NEXT: s_endpgm
;
@@ -346,29 +345,32 @@ define amdgpu_kernel void @fp_to_uint_v2f32_to_v2i64(ptr addrspace(1) %out, <2 x
; SI-LABEL: fp_to_uint_v2f32_to_v2i64:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s6, 0xcf800000
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s8, 0xcf800000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_trunc_f32_e32 v0, s5
-; SI-NEXT: v_trunc_f32_e32 v2, s4
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_trunc_f32_e32 v0, s3
+; SI-NEXT: v_trunc_f32_e32 v2, s2
; SI-NEXT: v_mul_f32_e32 v1, 0x2f800000, v0
; SI-NEXT: v_mul_f32_e32 v3, 0x2f800000, v2
; SI-NEXT: v_floor_f32_e32 v4, v1
; SI-NEXT: v_floor_f32_e32 v5, v3
; SI-NEXT: v_cvt_u32_f32_e32 v3, v4
; SI-NEXT: v_cvt_u32_f32_e32 v1, v5
-; SI-NEXT: v_fma_f32 v0, v4, s6, v0
-; SI-NEXT: v_fma_f32 v4, v5, s6, v2
+; SI-NEXT: v_fma_f32 v0, v4, s8, v0
+; SI-NEXT: v_fma_f32 v4, v5, s8, v2
; SI-NEXT: v_cvt_u32_f32_e32 v2, v0
; SI-NEXT: v_cvt_u32_f32_e32 v0, v4
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_uint_v2f32_to_v2i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_trunc_f32_e32 v0, s3
; VI-NEXT: v_trunc_f32_e32 v4, s2
@@ -383,9 +385,9 @@ define amdgpu_kernel void @fp_to_uint_v2f32_to_v2i64(ptr addrspace(1) %out, <2 x
; VI-NEXT: v_cvt_u32_f32_e32 v3, v5
; VI-NEXT: v_cvt_u32_f32_e32 v1, v6
; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_uint_v2f32_to_v2i64:
@@ -861,33 +863,33 @@ define amdgpu_kernel void @fp_to_uint_v4f32_to_v4i64(ptr addrspace(1) %out, <4 x
define amdgpu_kernel void @fp_to_uint_f32_to_i1(ptr addrspace(1) %out, float %in) #0 {
; SI-LABEL: fp_to_uint_f32_to_i1:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_eq_f32_e64 s[4:5], 1.0, s6
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_eq_f32_e64 s[0:1], 1.0, s2
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_uint_f32_to_i1:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_eq_f32_e64 s[4:5], 1.0, s6
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_cmp_eq_f32_e64 s[0:1], 1.0, s2
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_uint_f32_to_i1:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cmp_eq_f32_e64 s2, 1.0, s2
@@ -939,33 +941,33 @@ define amdgpu_kernel void @fp_to_uint_f32_to_i1(ptr addrspace(1) %out, float %in
define amdgpu_kernel void @fp_to_uint_fabs_f32_to_i1(ptr addrspace(1) %out, float %in) #0 {
; SI-LABEL: fp_to_uint_fabs_f32_to_i1:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cmp_eq_f32_e64 s[4:5], 1.0, |s6|
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_eq_f32_e64 s[0:1], 1.0, |s2|
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_uint_fabs_f32_to_i1:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_eq_f32_e64 s[4:5], 1.0, |s6|
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_cmp_eq_f32_e64 s[0:1], 1.0, |s2|
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_uint_fabs_f32_to_i1:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cmp_eq_f32_e64 s2, 1.0, |s2|
@@ -1018,21 +1020,21 @@ define amdgpu_kernel void @fp_to_uint_fabs_f32_to_i1(ptr addrspace(1) %out, floa
define amdgpu_kernel void @fp_to_uint_f32_to_i16(ptr addrspace(1) %out, float %in) #0 {
; SI-LABEL: fp_to_uint_f32_to_i16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_u32_f32_e32 v0, s6
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_u32_f32_e32 v0, s2
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_uint_f32_to_i16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_cvt_u32_f32_e32 v0, s2
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: buffer_store_short v0, off, s[0:3], 0
@@ -1040,9 +1042,7 @@ define amdgpu_kernel void @fp_to_uint_f32_to_i16(ptr addrspace(1) %out, float %i
;
; GFX11-SDAG-LABEL: fp_to_uint_f32_to_i16:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_u32_f32_e32 v0, s2
@@ -1092,28 +1092,31 @@ define amdgpu_kernel void @fp_to_uint_v2f32_to_v2i16(ptr addrspace(1) %out, <2 x
; SI-LABEL: fp_to_uint_v2f32_to_v2i16:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_cvt_u32_f32_e32 v0, s5
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_u32_f32_e32 v0, s3
; SI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; SI-NEXT: v_cvt_u32_f32_e32 v1, s4
+; SI-NEXT: v_cvt_u32_f32_e32 v1, s2
; SI-NEXT: v_or_b32_e32 v0, v1, v0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_uint_v2f32_to_v2i16:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s3
; VI-NEXT: v_cvt_u32_f32_sdwa v0, v0 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
; VI-NEXT: v_cvt_u32_f32_e32 v1, s2
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_or_b32_sdwa v0, v1, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_uint_v2f32_to_v2i16:
@@ -1166,28 +1169,31 @@ define amdgpu_kernel void @fp_to_uint_f32_to_v2i16(ptr addrspace(1) %out, float
; SI-LABEL: fp_to_uint_f32_to_v2i16:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_cvt_u32_f32_e32 v0, s4
-; SI-NEXT: v_cvt_u32_f32_e32 v1, s5
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_u32_f32_e32 v0, s2
+; SI-NEXT: v_cvt_u32_f32_e32 v1, s3
; SI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; SI-NEXT: v_or_b32_e32 v0, v0, v1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fp_to_uint_f32_to_v2i16:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: v_cvt_u32_f32_e32 v0, s2
; VI-NEXT: v_cvt_u32_f32_sdwa v1, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-SDAG-LABEL: fp_to_uint_f32_to_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/fpext.f16.ll b/llvm/test/CodeGen/AMDGPU/fpext.f16.ll
index fc7d87e92e1b3..cf63b5c55d6e4 100644
--- a/llvm/test/CodeGen/AMDGPU/fpext.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fpext.f16.ll
@@ -366,10 +366,9 @@ entry:
define amdgpu_kernel void @s_fneg_fpext_f16_to_f32(ptr addrspace(1) %r, i32 %a) {
; SI-LABEL: s_fneg_fpext_f16_to_f32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -377,10 +376,9 @@ define amdgpu_kernel void @s_fneg_fpext_f16_to_f32(ptr addrspace(1) %r, i32 %a)
;
; GFX89-LABEL: s_fneg_fpext_f16_to_f32:
; GFX89: ; %bb.0: ; %entry
-; GFX89-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX89-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX89-NEXT: s_mov_b32 s3, 0xf000
+; GFX89-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX89-NEXT: s_waitcnt lgkmcnt(0)
+; GFX89-NEXT: s_mov_b32 s3, 0xf000
; GFX89-NEXT: v_cvt_f32_f16_e32 v0, s2
; GFX89-NEXT: s_mov_b32 s2, -1
; GFX89-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -388,11 +386,9 @@ define amdgpu_kernel void @s_fneg_fpext_f16_to_f32(ptr addrspace(1) %r, i32 %a)
;
; GFX11-TRUE16-LABEL: s_fneg_fpext_f16_to_f32:
; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
@@ -400,11 +396,9 @@ define amdgpu_kernel void @s_fneg_fpext_f16_to_f32(ptr addrspace(1) %r, i32 %a)
;
; GFX11-FAKE16-LABEL: s_fneg_fpext_f16_to_f32:
; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v0, s2
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1
; GFX11-FAKE16-NEXT: buffer_store_b32 v0, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll b/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll
index db483562c46ab..2dd03d4726ada 100644
--- a/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptosi.f16.ll
@@ -583,36 +583,36 @@ entry:
define amdgpu_kernel void @fptosi_f16_to_i1(ptr addrspace(1) %out, half %in) {
; SI-LABEL: fptosi_f16_to_i1:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s6
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: v_cmp_eq_f32_e32 vcc, -1.0, v0
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fptosi_f16_to_i1:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_eq_f16_e64 s[4:5], -1.0, s6
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_cmp_eq_f16_e64 s[0:1], -1.0, s2
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: fptosi_f16_to_i1:
; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: v_cmp_eq_f16_e64 s2, -1.0, s2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2
@@ -622,11 +622,9 @@ define amdgpu_kernel void @fptosi_f16_to_i1(ptr addrspace(1) %out, half %in) {
;
; GFX11-FAKE16-LABEL: fptosi_f16_to_i1:
; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-FAKE16-NEXT: v_cmp_eq_f16_e64 s2, -1.0, s2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll b/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll
index 5b231eabad706..c5802690d0bf2 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui.f16.ll
@@ -580,37 +580,34 @@ entry:
define amdgpu_kernel void @fptoui_f16_to_i1(ptr addrspace(1) %out, half %in) {
; SI-LABEL: fptoui_f16_to_i1:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_cmp_eq_f32_e32 vcc, 1.0, v0
; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fptoui_f16_to_i1:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cmp_eq_f16_e64 s[4:5], 1.0, s6
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
-; VI-NEXT: buffer_store_byte v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_cmp_eq_f16_e64 s[0:1], 1.0, s2
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
+; VI-NEXT: buffer_store_byte v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: fptoui_f16_to_i1:
; GFX11-TRUE16: ; %bb.0: ; %entry
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: v_cmp_eq_f16_e64 s2, 1.0, s2
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2
@@ -620,11 +617,9 @@ define amdgpu_kernel void @fptoui_f16_to_i1(ptr addrspace(1) %out, half %in) {
;
; GFX11-FAKE16-LABEL: fptoui_f16_to_i1:
; GFX11-FAKE16: ; %bb.0: ; %entry
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-FAKE16-NEXT: v_cmp_eq_f16_e64 s2, 1.0, s2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, 0, 1, s2
diff --git a/llvm/test/CodeGen/AMDGPU/fptoui_uitofp.ll b/llvm/test/CodeGen/AMDGPU/fptoui_uitofp.ll
index a99cf8ef403d8..b8542f5ca6c93 100644
--- a/llvm/test/CodeGen/AMDGPU/fptoui_uitofp.ll
+++ b/llvm/test/CodeGen/AMDGPU/fptoui_uitofp.ll
@@ -5,19 +5,19 @@
define amdgpu_kernel void @fptoui_f32_to_i16_to_f32(ptr addrspace(1) %out, float %x) {
; GFX6-LABEL: fptoui_f32_to_i16_to_f32:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_trunc_f32_e64 v0, |s6|
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_trunc_f32_e64 v0, |s2|
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: fptoui_f32_to_i16_to_f32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_trunc_f32_e64 v1, |s2|
@@ -33,19 +33,19 @@ entry:
define amdgpu_kernel void @fptoui_f32_to_i32_to_f32(ptr addrspace(1) %out, float %x) {
; GFX6-LABEL: fptoui_f32_to_i32_to_f32:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_trunc_f32_e64 v0, |s6|
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_trunc_f32_e64 v0, |s2|
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: fptoui_f32_to_i32_to_f32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_trunc_f32_e64 v1, |s2|
@@ -61,19 +61,19 @@ entry:
define amdgpu_kernel void @fptoui_f32_to_i64_to_f32(ptr addrspace(1) %out, float %x) {
; GFX6-LABEL: fptoui_f32_to_i64_to_f32:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_trunc_f32_e64 v0, |s6|
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_trunc_f32_e64 v0, |s2|
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: fptoui_f32_to_i64_to_f32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_trunc_f32_e64 v1, |s2|
@@ -89,23 +89,20 @@ entry:
define amdgpu_kernel void @fptoui_f16_to_i16_to_f16(ptr addrspace(1) %out, half %x) {
; GFX6-LABEL: fptoui_f16_to_i16_to_f16:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s2
; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s0
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX6-NEXT: v_cvt_f32_u32_e32 v0, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: fptoui_f16_to_i16_to_f16:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_trunc_f16_e64 v1, |s2|
@@ -121,22 +118,19 @@ entry:
define amdgpu_kernel void @fptoui_f16_to_i32_to_f16(ptr addrspace(1) %out, half %x) {
; GFX6-LABEL: fptoui_f16_to_i32_to_f16:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: v_cvt_f32_f16_e64 v0, |s2|
; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cvt_f32_f16_e64 v0, |s0|
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: v_trunc_f32_e32 v0, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: fptoui_f16_to_i32_to_f16:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_trunc_f16_e64 v1, |s2|
@@ -152,22 +146,19 @@ entry:
define amdgpu_kernel void @fptoui_f16_to_i64_to_f16(ptr addrspace(1) %out, half %x) {
; GFX6-LABEL: fptoui_f16_to_i64_to_f16:
; GFX6: ; %bb.0: ; %entry
-; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: v_cvt_f32_f16_e64 v0, |s2|
; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cvt_f32_f16_e64 v0, |s0|
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GFX6-NEXT: v_trunc_f32_e32 v0, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
;
; GFX9-LABEL: fptoui_f16_to_i64_to_f16:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_trunc_f16_e64 v1, |s2|
diff --git a/llvm/test/CodeGen/AMDGPU/fshl.ll b/llvm/test/CodeGen/AMDGPU/fshl.ll
index 1233c1fe12f72..29d8ed56af5ab 100644
--- a/llvm/test/CodeGen/AMDGPU/fshl.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshl.ll
@@ -13,17 +13,19 @@ declare <4 x i32> @llvm.fshl.v4i32(<4 x i32>, <4 x i32>, <4 x i32>) nounwind rea
define amdgpu_kernel void @fshl_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z) {
; SI-LABEL: fshl_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dword s8, s[4:5], 0xd
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s8, s1
-; SI-NEXT: s_mov_b32 s9, s0
-; SI-NEXT: s_lshr_b32 s3, s0, 1
-; SI-NEXT: s_lshr_b64 s[0:1], s[8:9], 1
-; SI-NEXT: s_not_b32 s2, s2
-; SI-NEXT: s_mov_b32 s1, s3
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b32 s0, s3
+; SI-NEXT: s_mov_b32 s1, s2
+; SI-NEXT: s_lshr_b32 s9, s2, 1
+; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; SI-NEXT: s_not_b32 s2, s8
+; SI-NEXT: s_mov_b32 s1, s9
; SI-NEXT: s_and_b32 s2, s2, 31
; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
; SI-NEXT: v_mov_b32_e32 v0, s0
@@ -32,39 +34,39 @@ define amdgpu_kernel void @fshl_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
;
; VI-LABEL: fshl_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dword s4, s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s6, s1
-; VI-NEXT: s_mov_b32 s7, s0
-; VI-NEXT: s_lshr_b32 s3, s0, 1
-; VI-NEXT: s_not_b32 s2, s2
-; VI-NEXT: s_lshr_b64 s[0:1], s[6:7], 1
-; VI-NEXT: s_mov_b32 s1, s3
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_mov_b32 s0, s3
+; VI-NEXT: s_mov_b32 s1, s2
+; VI-NEXT: s_lshr_b32 s5, s2, 1
+; VI-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; VI-NEXT: s_not_b32 s2, s4
+; VI-NEXT: s_mov_b32 s1, s5
; VI-NEXT: s_and_b32 s2, s2, 31
; VI-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: v_mov_b32_e32 v1, s5
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fshl_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, s1
-; GFX9-NEXT: s_mov_b32 s5, s0
-; GFX9-NEXT: s_lshr_b32 s3, s0, 1
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
-; GFX9-NEXT: s_not_b32 s2, s2
-; GFX9-NEXT: s_mov_b32 s1, s3
-; GFX9-NEXT: s_and_b32 s2, s2, 31
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: global_store_dword v0, v1, s[6:7]
+; GFX9-NEXT: s_mov_b32 s4, s3
+; GFX9-NEXT: s_mov_b32 s5, s2
+; GFX9-NEXT: s_lshr_b32 s7, s2, 1
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[4:5], 1
+; GFX9-NEXT: s_not_b32 s4, s6
+; GFX9-NEXT: s_mov_b32 s3, s7
+; GFX9-NEXT: s_and_b32 s4, s4, 31
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; R600-LABEL: fshl_i32:
@@ -84,39 +86,39 @@ define amdgpu_kernel void @fshl_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX10-LABEL: fshl_i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s1
-; GFX10-NEXT: s_mov_b32 s5, s0
-; GFX10-NEXT: s_lshr_b32 s3, s0, 1
-; GFX10-NEXT: s_not_b32 s2, s2
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
-; GFX10-NEXT: s_mov_b32 s1, s3
-; GFX10-NEXT: s_and_b32 s2, s2, 31
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: global_store_dword v0, v1, s[6:7]
+; GFX10-NEXT: s_mov_b32 s4, s3
+; GFX10-NEXT: s_mov_b32 s5, s2
+; GFX10-NEXT: s_lshr_b32 s7, s2, 1
+; GFX10-NEXT: s_not_b32 s6, s6
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], 1
+; GFX10-NEXT: s_mov_b32 s3, s7
+; GFX10-NEXT: s_and_b32 s4, s6, 31
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fshl_i32:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s6, s1
-; GFX11-NEXT: s_mov_b32 s7, s0
-; GFX11-NEXT: s_lshr_b32 s3, s0, 1
-; GFX11-NEXT: s_not_b32 s2, s2
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[6:7], 1
-; GFX11-NEXT: s_mov_b32 s1, s3
-; GFX11-NEXT: s_and_b32 s2, s2, 31
+; GFX11-NEXT: s_mov_b32 s4, s3
+; GFX11-NEXT: s_mov_b32 s5, s2
+; GFX11-NEXT: s_lshr_b32 s7, s2, 1
+; GFX11-NEXT: s_not_b32 s6, s6
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[4:5], 1
+; GFX11-NEXT: s_mov_b32 s3, s7
+; GFX11-NEXT: s_and_b32 s4, s6, 31
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX11-NEXT: global_store_b32 v0, v1, s[4:5]
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_endpgm
entry:
%0 = call i32 @llvm.fshl.i32(i32 %x, i32 %y, i32 %z)
@@ -144,12 +146,12 @@ define amdgpu_kernel void @fshl_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s4, s3
-; VI-NEXT: s_mov_b32 s5, s2
-; VI-NEXT: s_lshr_b64 s[2:3], s[4:5], 25
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_mov_b32 s0, s3
+; VI-NEXT: s_mov_b32 s1, s2
+; VI-NEXT: s_lshr_b64 s[0:1], s[0:1], 25
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -209,85 +211,81 @@ entry:
define amdgpu_kernel void @fshl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y, <2 x i32> %z) {
; SI-LABEL: fshl_v2i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x9
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xf
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s11, 0xf000
; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s6, s3
-; SI-NEXT: s_mov_b32 s7, s1
-; SI-NEXT: s_lshr_b32 s12, s1, 1
-; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], 1
-; SI-NEXT: s_not_b32 s1, s5
-; SI-NEXT: s_mov_b32 s7, s12
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_mov_b32 s0, s5
+; SI-NEXT: s_mov_b32 s1, s3
+; SI-NEXT: s_lshr_b32 s12, s3, 1
+; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; SI-NEXT: s_not_b32 s3, s7
+; SI-NEXT: s_mov_b32 s1, s12
+; SI-NEXT: s_and_b32 s3, s3, 31
+; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
+; SI-NEXT: s_mov_b32 s5, s2
+; SI-NEXT: s_lshr_b32 s1, s2, 1
+; SI-NEXT: s_lshr_b64 s[2:3], s[4:5], 1
+; SI-NEXT: s_mov_b32 s3, s1
+; SI-NEXT: s_not_b32 s1, s6
; SI-NEXT: s_and_b32 s1, s1, 31
-; SI-NEXT: s_mov_b32 s3, s0
-; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], s1
-; SI-NEXT: s_lshr_b32 s5, s0, 1
-; SI-NEXT: s_lshr_b64 s[0:1], s[2:3], 1
-; SI-NEXT: s_not_b32 s2, s4
-; SI-NEXT: s_mov_b32 s1, s5
-; SI-NEXT: s_and_b32 s2, s2, 31
-; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s6
+; SI-NEXT: s_lshr_b64 s[2:3], s[2:3], s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fshl_v2i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x3c
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s8, s3
-; VI-NEXT: s_mov_b32 s9, s1
-; VI-NEXT: s_lshr_b32 s10, s1, 1
-; VI-NEXT: s_lshr_b64 s[8:9], s[8:9], 1
-; VI-NEXT: s_not_b32 s1, s5
-; VI-NEXT: s_mov_b32 s9, s10
-; VI-NEXT: s_and_b32 s1, s1, 31
-; VI-NEXT: s_mov_b32 s3, s0
-; VI-NEXT: s_lshr_b64 s[8:9], s[8:9], s1
-; VI-NEXT: s_lshr_b32 s5, s0, 1
-; VI-NEXT: s_lshr_b64 s[0:1], s[2:3], 1
-; VI-NEXT: s_not_b32 s2, s4
-; VI-NEXT: s_mov_b32 s1, s5
-; VI-NEXT: s_and_b32 s2, s2, 31
-; VI-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; VI-NEXT: v_mov_b32_e32 v2, s6
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s8
-; VI-NEXT: v_mov_b32_e32 v3, s7
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_mov_b32 s0, s5
+; VI-NEXT: s_mov_b32 s1, s3
+; VI-NEXT: s_lshr_b32 s8, s3, 1
+; VI-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; VI-NEXT: s_not_b32 s3, s7
+; VI-NEXT: s_mov_b32 s1, s8
+; VI-NEXT: s_and_b32 s3, s3, 31
+; VI-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
+; VI-NEXT: s_mov_b32 s5, s2
+; VI-NEXT: s_lshr_b32 s1, s2, 1
+; VI-NEXT: s_lshr_b64 s[2:3], s[4:5], 1
+; VI-NEXT: s_mov_b32 s3, s1
+; VI-NEXT: s_not_b32 s1, s6
+; VI-NEXT: s_and_b32 s1, s1, 31
+; VI-NEXT: s_lshr_b64 s[2:3], s[2:3], s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fshl_v2i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x3c
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, s3
-; GFX9-NEXT: s_mov_b32 s5, s1
-; GFX9-NEXT: s_lshr_b32 s10, s1, 1
-; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
-; GFX9-NEXT: s_not_b32 s1, s9
-; GFX9-NEXT: s_mov_b32 s5, s10
-; GFX9-NEXT: s_and_b32 s1, s1, 31
-; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s1
-; GFX9-NEXT: s_mov_b32 s3, s0
-; GFX9-NEXT: s_lshr_b32 s5, s0, 1
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[2:3], 1
-; GFX9-NEXT: s_not_b32 s2, s8
-; GFX9-NEXT: s_mov_b32 s1, s5
-; GFX9-NEXT: s_and_b32 s2, s2, 31
+; GFX9-NEXT: s_mov_b32 s0, s13
+; GFX9-NEXT: s_mov_b32 s1, s11
+; GFX9-NEXT: s_lshr_b32 s2, s11, 1
+; GFX9-NEXT: s_not_b32 s3, s15
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; GFX9-NEXT: s_mov_b32 s1, s2
+; GFX9-NEXT: s_and_b32 s2, s3, 31
; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: s_mov_b32 s13, s10
+; GFX9-NEXT: s_lshr_b32 s1, s10, 1
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[12:13], 1
+; GFX9-NEXT: s_mov_b32 s3, s1
+; GFX9-NEXT: s_not_b32 s1, s14
+; GFX9-NEXT: s_and_b32 s1, s1, 31
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
; GFX9-NEXT: s_endpgm
;
; R600-LABEL: fshl_v2i32:
@@ -310,27 +308,24 @@ define amdgpu_kernel void @fshl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
;
; GFX10-LABEL: fshl_v2i32:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX10-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s3
-; GFX10-NEXT: s_mov_b32 s5, s1
-; GFX10-NEXT: s_mov_b32 s3, s0
-; GFX10-NEXT: s_lshr_b32 s10, s1, 1
-; GFX10-NEXT: s_not_b32 s7, s7
-; GFX10-NEXT: s_lshr_b32 s11, s0, 1
-; GFX10-NEXT: s_not_b32 s6, s6
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[4:5], 1
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
-; GFX10-NEXT: s_and_b32 s4, s7, 31
-; GFX10-NEXT: s_and_b32 s5, s6, 31
-; GFX10-NEXT: s_mov_b32 s3, s11
-; GFX10-NEXT: s_mov_b32 s1, s10
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s5
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
+; GFX10-NEXT: s_mov_b32 s0, s13
+; GFX10-NEXT: s_mov_b32 s1, s11
+; GFX10-NEXT: s_not_b32 s2, s15
+; GFX10-NEXT: s_mov_b32 s13, s10
+; GFX10-NEXT: s_lshr_b32 s4, s11, 1
+; GFX10-NEXT: s_lshr_b32 s5, s10, 1
+; GFX10-NEXT: s_not_b32 s6, s14
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], 1
+; GFX10-NEXT: s_and_b32 s7, s2, 31
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[12:13], 1
+; GFX10-NEXT: s_and_b32 s6, s6, 31
+; GFX10-NEXT: s_mov_b32 s3, s5
+; GFX10-NEXT: s_mov_b32 s1, s4
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s6
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s7
; GFX10-NEXT: v_mov_b32_e32 v0, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
@@ -338,30 +333,27 @@ define amdgpu_kernel void @fshl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
;
; GFX11-LABEL: fshl_v2i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s8, s3
-; GFX11-NEXT: s_mov_b32 s9, s1
-; GFX11-NEXT: s_mov_b32 s3, s0
-; GFX11-NEXT: s_lshr_b32 s10, s1, 1
+; GFX11-NEXT: s_mov_b32 s8, s5
+; GFX11-NEXT: s_mov_b32 s9, s3
+; GFX11-NEXT: s_mov_b32 s5, s2
+; GFX11-NEXT: s_lshr_b32 s10, s3, 1
; GFX11-NEXT: s_not_b32 s7, s7
-; GFX11-NEXT: s_lshr_b32 s11, s0, 1
+; GFX11-NEXT: s_lshr_b32 s11, s2, 1
; GFX11-NEXT: s_not_b32 s6, s6
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[8:9], 1
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], 1
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[8:9], 1
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], 1
; GFX11-NEXT: s_and_b32 s7, s7, 31
; GFX11-NEXT: s_and_b32 s6, s6, 31
-; GFX11-NEXT: s_mov_b32 s3, s11
-; GFX11-NEXT: s_mov_b32 s1, s10
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s6
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s7
+; GFX11-NEXT: s_mov_b32 s5, s11
+; GFX11-NEXT: s_mov_b32 s3, s10
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s7
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s0
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
entry:
%0 = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %z)
@@ -372,52 +364,54 @@ entry:
define amdgpu_kernel void @fshl_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y) {
; SI-LABEL: fshl_v2i32_imm:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s8, s3
-; SI-NEXT: s_mov_b32 s9, s1
-; SI-NEXT: s_mov_b32 s3, s0
-; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 23
-; SI-NEXT: s_lshr_b64 s[0:1], s[2:3], 25
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s8
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b32 s0, s9
+; SI-NEXT: s_mov_b32 s1, s3
+; SI-NEXT: s_mov_b32 s9, s2
+; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], 23
+; SI-NEXT: s_lshr_b64 s[2:3], s[8:9], 25
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fshl_v2i32_imm:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s6, s3
-; VI-NEXT: s_mov_b32 s7, s1
-; VI-NEXT: s_mov_b32 s3, s0
-; VI-NEXT: s_lshr_b64 s[0:1], s[6:7], 23
-; VI-NEXT: s_lshr_b64 s[2:3], s[2:3], 25
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v1, s0
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_mov_b32 s0, s5
+; VI-NEXT: s_mov_b32 s1, s3
+; VI-NEXT: s_mov_b32 s5, s2
+; VI-NEXT: s_lshr_b64 s[0:1], s[0:1], 23
+; VI-NEXT: s_lshr_b64 s[2:3], s[4:5], 25
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fshl_v2i32_imm:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, s3
-; GFX9-NEXT: s_mov_b32 s5, s1
-; GFX9-NEXT: s_mov_b32 s3, s0
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[4:5], 23
-; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], 25
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: s_mov_b32 s4, s7
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: s_mov_b32 s7, s2
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[4:5], 23
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[6:7], 25
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; R600-LABEL: fshl_v2i32_imm:
@@ -437,35 +431,35 @@ define amdgpu_kernel void @fshl_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
; GFX10-LABEL: fshl_v2i32_imm:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s3
-; GFX10-NEXT: s_mov_b32 s3, s0
-; GFX10-NEXT: s_mov_b32 s5, s1
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[2:3], 25
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], 23
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX10-NEXT: s_mov_b32 s4, s7
+; GFX10-NEXT: s_mov_b32 s7, s2
+; GFX10-NEXT: s_mov_b32 s5, s3
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[6:7], 25
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], 23
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fshl_v2i32_imm:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s6, s3
-; GFX11-NEXT: s_mov_b32 s3, s0
-; GFX11-NEXT: s_mov_b32 s7, s1
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[2:3], 25
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[6:7], 23
+; GFX11-NEXT: s_mov_b32 s4, s7
+; GFX11-NEXT: s_mov_b32 s7, s2
+; GFX11-NEXT: s_mov_b32 s5, s3
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[6:7], 25
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], 23
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
entry:
%0 = call <2 x i32> @llvm.fshl.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 7, i32 9>)
@@ -875,27 +869,28 @@ define amdgpu_kernel void @orxor2or1(ptr addrspace(1) %in, i32 %a, i32 %b) {
; SI-LABEL: orxor2or1:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_lshl_b32 s6, s4, 7
-; SI-NEXT: s_or_b32 s6, s5, s6
-; SI-NEXT: s_cselect_b32 s4, s5, s4
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_lshl_b32 s0, s2, 7
+; SI-NEXT: s_or_b32 s0, s3, s0
+; SI-NEXT: s_cselect_b32 s0, s3, s2
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: orxor2or1:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b32 s4, s2, 7
-; VI-NEXT: s_or_b32 s4, s3, s4
-; VI-NEXT: s_cselect_b32 s2, s3, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_lshl_b32 s0, s2, 7
+; VI-NEXT: s_or_b32 s0, s3, s0
+; VI-NEXT: s_cselect_b32 s0, s3, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fshr.ll b/llvm/test/CodeGen/AMDGPU/fshr.ll
index be49838f86417..e7a8291d9820f 100644
--- a/llvm/test/CodeGen/AMDGPU/fshr.ll
+++ b/llvm/test/CodeGen/AMDGPU/fshr.ll
@@ -25,46 +25,48 @@ declare <2 x i24> @llvm.fshr.v2i24(<2 x i24>, <2 x i24>, <2 x i24>)
define amdgpu_kernel void @fshr_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z) {
; SI-LABEL: fshr_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dword s8, s[4:5], 0xd
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s8, s1
-; SI-NEXT: s_mov_b32 s9, s0
-; SI-NEXT: s_and_b32 s0, s2, 31
-; SI-NEXT: s_lshr_b64 s[0:1], s[8:9], s0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b32 s0, s3
+; SI-NEXT: s_mov_b32 s1, s2
+; SI-NEXT: s_and_b32 s2, s8, 31
+; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fshr_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dword s4, s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s6, s1
-; VI-NEXT: s_mov_b32 s7, s0
-; VI-NEXT: s_and_b32 s0, s2, 31
-; VI-NEXT: s_lshr_b64 s[0:1], s[6:7], s0
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: v_mov_b32_e32 v1, s5
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_mov_b32 s0, s3
+; VI-NEXT: s_mov_b32 s1, s2
+; VI-NEXT: s_and_b32 s2, s4, 31
+; VI-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fshr_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, s1
-; GFX9-NEXT: s_mov_b32 s5, s0
-; GFX9-NEXT: s_and_b32 s0, s2, 31
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[4:5], s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: global_store_dword v0, v1, s[6:7]
+; GFX9-NEXT: s_mov_b32 s4, s3
+; GFX9-NEXT: s_mov_b32 s5, s2
+; GFX9-NEXT: s_and_b32 s2, s6, 31
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; R600-LABEL: fshr_i32:
@@ -81,46 +83,46 @@ define amdgpu_kernel void @fshr_i32(ptr addrspace(1) %in, i32 %x, i32 %y, i32 %z
; GFX10-LABEL: fshr_i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s1
-; GFX10-NEXT: s_mov_b32 s5, s0
-; GFX10-NEXT: s_and_b32 s0, s2, 31
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[4:5], s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s0
-; GFX10-NEXT: global_store_dword v0, v1, s[6:7]
+; GFX10-NEXT: s_mov_b32 s4, s3
+; GFX10-NEXT: s_mov_b32 s5, s2
+; GFX10-NEXT: s_and_b32 s2, s6, 31
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fshr_i32:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s6, s1
-; GFX11-NEXT: s_mov_b32 s7, s0
-; GFX11-NEXT: s_and_b32 s0, s2, 31
+; GFX11-NEXT: s_mov_b32 s4, s3
+; GFX11-NEXT: s_mov_b32 s5, s2
+; GFX11-NEXT: s_and_b32 s2, s6, 31
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[6:7], s0
-; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX11-NEXT: global_store_b32 v0, v1, s[4:5]
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
+; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: fshr_i32:
; GFX12: ; %bb.0: ; %entry
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c
-; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_load_b32 s6, s[4:5], 0x34
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s6, s1
-; GFX12-NEXT: s_mov_b32 s7, s0
-; GFX12-NEXT: s_and_b32 s0, s2, 31
+; GFX12-NEXT: s_mov_b32 s4, s3
+; GFX12-NEXT: s_mov_b32 s5, s2
+; GFX12-NEXT: s_and_b32 s2, s6, 31
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_lshr_b64 s[0:1], s[6:7], s0
-; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX12-NEXT: global_store_b32 v0, v1, s[4:5]
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12-NEXT: s_endpgm
entry:
%0 = call i32 @llvm.fshr.i32(i32 %x, i32 %y, i32 %z)
@@ -148,12 +150,12 @@ define amdgpu_kernel void @fshr_i32_imm(ptr addrspace(1) %in, i32 %x, i32 %y) {
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s4, s3
-; VI-NEXT: s_mov_b32 s5, s2
-; VI-NEXT: s_lshr_b64 s[2:3], s[4:5], 7
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_mov_b32 s0, s3
+; VI-NEXT: s_mov_b32 s1, s2
+; VI-NEXT: s_lshr_b64 s[0:1], s[0:1], 7
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -244,12 +246,12 @@ define amdgpu_kernel void @fshr_i32_imm_src0(ptr addrspace(1) %in, i32 %x, i32 %
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_mov_b32 s5, 7
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s4, s3
-; VI-NEXT: s_and_b32 s2, s2, 31
-; VI-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_mov_b32 s4, s3
+; VI-NEXT: s_and_b32 s0, s2, 31
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_lshr_b64 s[0:1], s[4:5], s0
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -325,60 +327,56 @@ entry:
define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y, <2 x i32> %z) {
; SI-LABEL: fshr_v2i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x9
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xf
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s11, 0xf000
; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s6, s3
-; SI-NEXT: s_mov_b32 s7, s1
-; SI-NEXT: s_and_b32 s1, s5, 31
-; SI-NEXT: s_mov_b32 s3, s0
-; SI-NEXT: s_and_b32 s0, s4, 31
-; SI-NEXT: s_lshr_b64 s[6:7], s[6:7], s1
-; SI-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s6
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_mov_b32 s0, s5
+; SI-NEXT: s_mov_b32 s1, s3
+; SI-NEXT: s_and_b32 s3, s7, 31
+; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
+; SI-NEXT: s_mov_b32 s5, s2
+; SI-NEXT: s_and_b32 s1, s6, 31
+; SI-NEXT: s_lshr_b64 s[2:3], s[4:5], s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fshr_v2i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s8, s3
-; VI-NEXT: s_mov_b32 s9, s1
-; VI-NEXT: s_and_b32 s1, s7, 31
-; VI-NEXT: s_mov_b32 s3, s0
-; VI-NEXT: s_and_b32 s0, s6, 31
-; VI-NEXT: s_lshr_b64 s[8:9], s[8:9], s1
-; VI-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
-; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s8
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_mov_b32 s0, s5
+; VI-NEXT: s_mov_b32 s1, s3
+; VI-NEXT: s_and_b32 s3, s7, 31
+; VI-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
+; VI-NEXT: s_mov_b32 s5, s2
+; VI-NEXT: s_and_b32 s1, s6, 31
+; VI-NEXT: s_lshr_b64 s[2:3], s[4:5], s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fshr_v2i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, s3
-; GFX9-NEXT: s_mov_b32 s5, s1
-; GFX9-NEXT: s_and_b32 s1, s7, 31
-; GFX9-NEXT: s_mov_b32 s3, s0
-; GFX9-NEXT: s_and_b32 s0, s6, 31
-; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s1
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-NEXT: s_mov_b32 s0, s13
+; GFX9-NEXT: s_mov_b32 s1, s11
+; GFX9-NEXT: s_and_b32 s2, s15, 31
+; GFX9-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
+; GFX9-NEXT: s_mov_b32 s13, s10
+; GFX9-NEXT: s_and_b32 s1, s14, 31
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[12:13], s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
; GFX9-NEXT: s_endpgm
;
@@ -398,62 +396,53 @@ define amdgpu_kernel void @fshr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
;
; GFX10-LABEL: fshr_v2i32:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX10-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s3
-; GFX10-NEXT: s_mov_b32 s5, s1
-; GFX10-NEXT: s_mov_b32 s3, s0
-; GFX10-NEXT: s_and_b32 s0, s6, 31
-; GFX10-NEXT: s_and_b32 s6, s7, 31
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], s6
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: s_mov_b32 s0, s13
+; GFX10-NEXT: s_mov_b32 s1, s11
+; GFX10-NEXT: s_mov_b32 s13, s10
+; GFX10-NEXT: s_and_b32 s2, s14, 31
+; GFX10-NEXT: s_and_b32 s4, s15, 31
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[12:13], s2
+; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s0
; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[8:9]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fshr_v2i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s8, s3
-; GFX11-NEXT: s_mov_b32 s9, s1
-; GFX11-NEXT: s_mov_b32 s3, s0
-; GFX11-NEXT: s_and_b32 s0, s6, 31
+; GFX11-NEXT: s_mov_b32 s8, s5
+; GFX11-NEXT: s_mov_b32 s9, s3
+; GFX11-NEXT: s_mov_b32 s5, s2
+; GFX11-NEXT: s_and_b32 s2, s6, 31
; GFX11-NEXT: s_and_b32 s6, s7, 31
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[8:9], s6
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[8:9], s6
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: fshr_v2i32:
; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_clause 0x2
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
-; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s8, s3
-; GFX12-NEXT: s_mov_b32 s9, s1
-; GFX12-NEXT: s_mov_b32 s3, s0
-; GFX12-NEXT: s_and_b32 s0, s6, 31
+; GFX12-NEXT: s_mov_b32 s8, s5
+; GFX12-NEXT: s_mov_b32 s9, s3
+; GFX12-NEXT: s_mov_b32 s5, s2
+; GFX12-NEXT: s_and_b32 s2, s6, 31
; GFX12-NEXT: s_and_b32 s6, s7, 31
-; GFX12-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
-; GFX12-NEXT: s_lshr_b64 s[2:3], s[8:9], s6
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
+; GFX12-NEXT: s_lshr_b64 s[4:5], s[8:9], s6
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
entry:
%0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> %z)
@@ -464,52 +453,54 @@ entry:
define amdgpu_kernel void @fshr_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y) {
; SI-LABEL: fshr_v2i32_imm:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s8, s3
-; SI-NEXT: s_mov_b32 s9, s1
-; SI-NEXT: s_mov_b32 s3, s0
-; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], 9
-; SI-NEXT: s_lshr_b64 s[0:1], s[2:3], 7
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s8
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b32 s0, s9
+; SI-NEXT: s_mov_b32 s1, s3
+; SI-NEXT: s_mov_b32 s9, s2
+; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], 9
+; SI-NEXT: s_lshr_b64 s[2:3], s[8:9], 7
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fshr_v2i32_imm:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s6, s3
-; VI-NEXT: s_mov_b32 s7, s1
-; VI-NEXT: s_mov_b32 s3, s0
-; VI-NEXT: s_lshr_b64 s[0:1], s[6:7], 9
-; VI-NEXT: s_lshr_b64 s[2:3], s[2:3], 7
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v1, s0
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_mov_b32 s0, s5
+; VI-NEXT: s_mov_b32 s1, s3
+; VI-NEXT: s_mov_b32 s5, s2
+; VI-NEXT: s_lshr_b64 s[0:1], s[0:1], 9
+; VI-NEXT: s_lshr_b64 s[2:3], s[4:5], 7
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fshr_v2i32_imm:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s4, s3
-; GFX9-NEXT: s_mov_b32 s5, s1
-; GFX9-NEXT: s_mov_b32 s3, s0
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[4:5], 9
-; GFX9-NEXT: s_lshr_b64 s[2:3], s[2:3], 7
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: s_mov_b32 s4, s7
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: s_mov_b32 s7, s2
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[4:5], 9
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[6:7], 7
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; R600-LABEL: fshr_v2i32_imm:
@@ -529,52 +520,52 @@ define amdgpu_kernel void @fshr_v2i32_imm(ptr addrspace(1) %in, <2 x i32> %x, <2
; GFX10-LABEL: fshr_v2i32_imm:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s3
-; GFX10-NEXT: s_mov_b32 s3, s0
-; GFX10-NEXT: s_mov_b32 s5, s1
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[2:3], 7
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], 9
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX10-NEXT: s_mov_b32 s4, s7
+; GFX10-NEXT: s_mov_b32 s7, s2
+; GFX10-NEXT: s_mov_b32 s5, s3
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[6:7], 7
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], 9
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fshr_v2i32_imm:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s6, s3
-; GFX11-NEXT: s_mov_b32 s3, s0
-; GFX11-NEXT: s_mov_b32 s7, s1
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[2:3], 7
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[6:7], 9
+; GFX11-NEXT: s_mov_b32 s4, s7
+; GFX11-NEXT: s_mov_b32 s7, s2
+; GFX11-NEXT: s_mov_b32 s5, s3
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[6:7], 7
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], 9
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: fshr_v2i32_imm:
; GFX12: ; %bb.0: ; %entry
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s6, s3
-; GFX12-NEXT: s_mov_b32 s3, s0
-; GFX12-NEXT: s_mov_b32 s7, s1
-; GFX12-NEXT: s_lshr_b64 s[0:1], s[2:3], 7
-; GFX12-NEXT: s_lshr_b64 s[2:3], s[6:7], 9
+; GFX12-NEXT: s_mov_b32 s4, s7
+; GFX12-NEXT: s_mov_b32 s7, s2
+; GFX12-NEXT: s_mov_b32 s5, s3
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[6:7], 7
+; GFX12-NEXT: s_lshr_b64 s[4:5], s[4:5], 9
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
entry:
%0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i32> <i32 7, i32 9>)
@@ -585,61 +576,63 @@ entry:
define amdgpu_kernel void @fshr_v2i32_imm_src1(ptr addrspace(1) %in, <2 x i32> %x, <2 x i32> %y) {
; SI-LABEL: fshr_v2i32_imm_src1:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s8, 9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; SI-NEXT: s_mov_b32 s10, 7
; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s9, s1
-; SI-NEXT: s_and_b32 s1, s3, 31
-; SI-NEXT: s_mov_b32 s11, s0
-; SI-NEXT: s_and_b32 s0, s2, 31
-; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], s1
-; SI-NEXT: s_lshr_b64 s[0:1], s[10:11], s0
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s8
+; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_mov_b32 s0, 9
+; SI-NEXT: s_mov_b32 s1, s3
+; SI-NEXT: s_and_b32 s3, s9, 31
+; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
+; SI-NEXT: s_mov_b32 s11, s2
+; SI-NEXT: s_and_b32 s1, s8, 31
+; SI-NEXT: s_lshr_b64 s[2:3], s[10:11], s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: fshr_v2i32_imm_src1:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; VI-NEXT: s_mov_b32 s6, 9
-; VI-NEXT: s_mov_b32 s8, 7
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s7, s1
-; VI-NEXT: s_and_b32 s1, s3, 31
-; VI-NEXT: s_mov_b32 s9, s0
-; VI-NEXT: s_and_b32 s0, s2, 31
-; VI-NEXT: s_lshr_b64 s[6:7], s[6:7], s1
-; VI-NEXT: s_lshr_b64 s[0:1], s[8:9], s0
-; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s6
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_mov_b32 s7, s3
+; VI-NEXT: s_and_b32 s0, s5, 31
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_lshr_b64 s[0:1], s[6:7], s0
+; VI-NEXT: s_mov_b32 s6, 7
+; VI-NEXT: s_mov_b32 s7, s2
+; VI-NEXT: s_and_b32 s1, s4, 31
+; VI-NEXT: s_lshr_b64 s[2:3], s[6:7], s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: fshr_v2i32_imm_src1:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX9-NEXT: s_mov_b32 s4, 9
; GFX9-NEXT: s_mov_b32 s8, 7
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s5, s1
-; GFX9-NEXT: s_and_b32 s1, s3, 31
-; GFX9-NEXT: s_mov_b32 s9, s0
-; GFX9-NEXT: s_and_b32 s0, s2, 31
-; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s1
-; GFX9-NEXT: s_lshr_b64 s[0:1], s[8:9], s0
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: s_mov_b32 s5, s3
+; GFX9-NEXT: s_and_b32 s3, s7, 31
+; GFX9-NEXT: s_mov_b32 s9, s2
+; GFX9-NEXT: s_and_b32 s2, s6, 31
+; GFX9-NEXT: s_lshr_b64 s[4:5], s[4:5], s3
+; GFX9-NEXT: s_lshr_b64 s[2:3], s[8:9], s2
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; R600-LABEL: fshr_v2i32_imm_src1:
@@ -659,61 +652,61 @@ define amdgpu_kernel void @fshr_v2i32_imm_src1(ptr addrspace(1) %in, <2 x i32> %
; GFX10-LABEL: fshr_v2i32_imm_src1:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX10-NEXT: s_mov_b32 s4, 9
; GFX10-NEXT: s_mov_b32 s8, 7
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s5, s1
-; GFX10-NEXT: s_mov_b32 s9, s0
-; GFX10-NEXT: s_and_b32 s0, s2, 31
-; GFX10-NEXT: s_and_b32 s2, s3, 31
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[8:9], s0
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], s2
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX10-NEXT: s_mov_b32 s5, s3
+; GFX10-NEXT: s_mov_b32 s9, s2
+; GFX10-NEXT: s_and_b32 s2, s6, 31
+; GFX10-NEXT: s_and_b32 s6, s7, 31
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[8:9], s2
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s6
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fshr_v2i32_imm_src1:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX11-NEXT: s_mov_b32 s6, 9
; GFX11-NEXT: s_mov_b32 s8, 7
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s7, s1
-; GFX11-NEXT: s_mov_b32 s9, s0
-; GFX11-NEXT: s_and_b32 s0, s2, 31
-; GFX11-NEXT: s_and_b32 s2, s3, 31
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[8:9], s0
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[6:7], s2
+; GFX11-NEXT: s_mov_b32 s7, s3
+; GFX11-NEXT: s_mov_b32 s9, s2
+; GFX11-NEXT: s_and_b32 s2, s4, 31
+; GFX11-NEXT: s_and_b32 s4, s5, 31
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[8:9], s2
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[6:7], s4
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: fshr_v2i32_imm_src1:
; GFX12: ; %bb.0: ; %entry
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT: s_mov_b32 s6, 9
; GFX12-NEXT: s_mov_b32 s8, 7
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_mov_b32 s7, s1
-; GFX12-NEXT: s_mov_b32 s9, s0
-; GFX12-NEXT: s_and_b32 s0, s2, 31
-; GFX12-NEXT: s_and_b32 s2, s3, 31
-; GFX12-NEXT: s_lshr_b64 s[0:1], s[8:9], s0
-; GFX12-NEXT: s_lshr_b64 s[2:3], s[6:7], s2
+; GFX12-NEXT: s_mov_b32 s7, s3
+; GFX12-NEXT: s_mov_b32 s9, s2
+; GFX12-NEXT: s_and_b32 s2, s4, 31
+; GFX12-NEXT: s_and_b32 s4, s5, 31
+; GFX12-NEXT: s_lshr_b64 s[2:3], s[8:9], s2
+; GFX12-NEXT: s_lshr_b64 s[4:5], s[6:7], s4
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-NEXT: v_mov_b32_e32 v0, s0
-; GFX12-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
entry:
%0 = call <2 x i32> @llvm.fshr.v2i32(<2 x i32> %x, <2 x i32> <i32 7, i32 9>, <2 x i32> %y)
diff --git a/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll b/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
index 4fe58ab538a7e..92dece0ad812c 100644
--- a/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/fsqrt.f32.ll
@@ -3968,18 +3968,19 @@ define float @v_elim_redun_check_ult_sqrt_ulp3(float %in) {
define amdgpu_kernel void @elim_redun_check_neg0(ptr addrspace(1) %out, float %in) {
; SDAG-IEEE-LABEL: elim_redun_check_neg0:
; SDAG-IEEE: ; %bb.0: ; %entry
-; SDAG-IEEE-NEXT: s_load_dword s0, s[4:5], 0xb
-; SDAG-IEEE-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SDAG-IEEE-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SDAG-IEEE-NEXT: v_mov_b32_e32 v0, 0xf800000
; SDAG-IEEE-NEXT: v_mov_b32_e32 v1, 0x4f800000
-; SDAG-IEEE-NEXT: s_mov_b32 s7, 0xf000
+; SDAG-IEEE-NEXT: s_mov_b32 s11, 0xf000
+; SDAG-IEEE-NEXT: s_mov_b32 s10, -1
; SDAG-IEEE-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-IEEE-NEXT: v_mul_f32_e32 v1, s0, v1
-; SDAG-IEEE-NEXT: v_mov_b32_e32 v2, s0
-; SDAG-IEEE-NEXT: v_cmp_lt_f32_e64 s[0:1], s0, v0
+; SDAG-IEEE-NEXT: v_mul_f32_e32 v1, s6, v1
+; SDAG-IEEE-NEXT: v_mov_b32_e32 v2, s6
+; SDAG-IEEE-NEXT: v_cmp_lt_f32_e64 s[0:1], s6, v0
; SDAG-IEEE-NEXT: v_cndmask_b32_e64 v0, v2, v1, s[0:1]
; SDAG-IEEE-NEXT: v_sqrt_f32_e32 v1, v0
-; SDAG-IEEE-NEXT: s_mov_b32 s6, -1
+; SDAG-IEEE-NEXT: s_mov_b32 s8, s4
+; SDAG-IEEE-NEXT: s_mov_b32 s9, s5
; SDAG-IEEE-NEXT: v_add_i32_e32 v2, vcc, -1, v1
; SDAG-IEEE-NEXT: v_fma_f32 v3, -v2, v1, v0
; SDAG-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v3
@@ -3993,7 +3994,7 @@ define amdgpu_kernel void @elim_redun_check_neg0(ptr addrspace(1) %out, float %i
; SDAG-IEEE-NEXT: v_mov_b32_e32 v2, 0x260
; SDAG-IEEE-NEXT: v_cmp_class_f32_e32 vcc, v0, v2
; SDAG-IEEE-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-IEEE-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SDAG-IEEE-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SDAG-IEEE-NEXT: s_endpgm
;
; GISEL-IEEE-LABEL: elim_redun_check_neg0:
@@ -4045,18 +4046,17 @@ define amdgpu_kernel void @elim_redun_check_neg0(ptr addrspace(1) %out, float %i
;
; SDAG-DAZ-LABEL: elim_redun_check_neg0:
; SDAG-DAZ: ; %bb.0: ; %entry
-; SDAG-DAZ-NEXT: s_load_dword s0, s[4:5], 0xb
+; SDAG-DAZ-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SDAG-DAZ-NEXT: v_mov_b32_e32 v0, 0xf800000
; SDAG-DAZ-NEXT: v_mov_b32_e32 v1, 0x4f800000
-; SDAG-DAZ-NEXT: s_mov_b32 s3, 0xf000
-; SDAG-DAZ-NEXT: s_mov_b32 s2, -1
; SDAG-DAZ-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-DAZ-NEXT: v_mul_f32_e32 v1, s0, v1
-; SDAG-DAZ-NEXT: v_mov_b32_e32 v2, s0
-; SDAG-DAZ-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; SDAG-DAZ-NEXT: s_mov_b32 s3, 0xf000
+; SDAG-DAZ-NEXT: v_mul_f32_e32 v1, s2, v1
+; SDAG-DAZ-NEXT: v_mov_b32_e32 v2, s2
+; SDAG-DAZ-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; SDAG-DAZ-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; SDAG-DAZ-NEXT: v_rsq_f32_e32 v1, v0
-; SDAG-DAZ-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SDAG-DAZ-NEXT: s_mov_b32 s2, -1
; SDAG-DAZ-NEXT: v_mul_f32_e32 v2, v0, v1
; SDAG-DAZ-NEXT: v_mul_f32_e32 v1, 0.5, v1
; SDAG-DAZ-NEXT: v_fma_f32 v3, -v1, v2, 0.5
@@ -4069,7 +4069,6 @@ define amdgpu_kernel void @elim_redun_check_neg0(ptr addrspace(1) %out, float %i
; SDAG-DAZ-NEXT: v_mov_b32_e32 v2, 0x260
; SDAG-DAZ-NEXT: v_cmp_class_f32_e32 vcc, v0, v2
; SDAG-DAZ-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-DAZ-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-DAZ-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SDAG-DAZ-NEXT: s_endpgm
;
@@ -4122,18 +4121,19 @@ entry:
define amdgpu_kernel void @elim_redun_check_pos0(ptr addrspace(1) %out, float %in) {
; SDAG-IEEE-LABEL: elim_redun_check_pos0:
; SDAG-IEEE: ; %bb.0: ; %entry
-; SDAG-IEEE-NEXT: s_load_dword s0, s[4:5], 0xb
-; SDAG-IEEE-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SDAG-IEEE-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SDAG-IEEE-NEXT: v_mov_b32_e32 v0, 0xf800000
; SDAG-IEEE-NEXT: v_mov_b32_e32 v1, 0x4f800000
-; SDAG-IEEE-NEXT: s_mov_b32 s7, 0xf000
+; SDAG-IEEE-NEXT: s_mov_b32 s11, 0xf000
+; SDAG-IEEE-NEXT: s_mov_b32 s10, -1
; SDAG-IEEE-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-IEEE-NEXT: v_mul_f32_e32 v1, s0, v1
-; SDAG-IEEE-NEXT: v_mov_b32_e32 v2, s0
-; SDAG-IEEE-NEXT: v_cmp_lt_f32_e64 s[0:1], s0, v0
+; SDAG-IEEE-NEXT: v_mul_f32_e32 v1, s6, v1
+; SDAG-IEEE-NEXT: v_mov_b32_e32 v2, s6
+; SDAG-IEEE-NEXT: v_cmp_lt_f32_e64 s[0:1], s6, v0
; SDAG-IEEE-NEXT: v_cndmask_b32_e64 v0, v2, v1, s[0:1]
; SDAG-IEEE-NEXT: v_sqrt_f32_e32 v1, v0
-; SDAG-IEEE-NEXT: s_mov_b32 s6, -1
+; SDAG-IEEE-NEXT: s_mov_b32 s8, s4
+; SDAG-IEEE-NEXT: s_mov_b32 s9, s5
; SDAG-IEEE-NEXT: v_add_i32_e32 v2, vcc, -1, v1
; SDAG-IEEE-NEXT: v_fma_f32 v3, -v2, v1, v0
; SDAG-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v3
@@ -4147,7 +4147,7 @@ define amdgpu_kernel void @elim_redun_check_pos0(ptr addrspace(1) %out, float %i
; SDAG-IEEE-NEXT: v_mov_b32_e32 v2, 0x260
; SDAG-IEEE-NEXT: v_cmp_class_f32_e32 vcc, v0, v2
; SDAG-IEEE-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-IEEE-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SDAG-IEEE-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SDAG-IEEE-NEXT: s_endpgm
;
; GISEL-IEEE-LABEL: elim_redun_check_pos0:
@@ -4198,18 +4198,17 @@ define amdgpu_kernel void @elim_redun_check_pos0(ptr addrspace(1) %out, float %i
;
; SDAG-DAZ-LABEL: elim_redun_check_pos0:
; SDAG-DAZ: ; %bb.0: ; %entry
-; SDAG-DAZ-NEXT: s_load_dword s0, s[4:5], 0xb
+; SDAG-DAZ-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SDAG-DAZ-NEXT: v_mov_b32_e32 v0, 0xf800000
; SDAG-DAZ-NEXT: v_mov_b32_e32 v1, 0x4f800000
-; SDAG-DAZ-NEXT: s_mov_b32 s3, 0xf000
-; SDAG-DAZ-NEXT: s_mov_b32 s2, -1
; SDAG-DAZ-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-DAZ-NEXT: v_mul_f32_e32 v1, s0, v1
-; SDAG-DAZ-NEXT: v_mov_b32_e32 v2, s0
-; SDAG-DAZ-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; SDAG-DAZ-NEXT: s_mov_b32 s3, 0xf000
+; SDAG-DAZ-NEXT: v_mul_f32_e32 v1, s2, v1
+; SDAG-DAZ-NEXT: v_mov_b32_e32 v2, s2
+; SDAG-DAZ-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; SDAG-DAZ-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; SDAG-DAZ-NEXT: v_rsq_f32_e32 v1, v0
-; SDAG-DAZ-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SDAG-DAZ-NEXT: s_mov_b32 s2, -1
; SDAG-DAZ-NEXT: v_mul_f32_e32 v2, v0, v1
; SDAG-DAZ-NEXT: v_mul_f32_e32 v1, 0.5, v1
; SDAG-DAZ-NEXT: v_fma_f32 v3, -v1, v2, 0.5
@@ -4222,7 +4221,6 @@ define amdgpu_kernel void @elim_redun_check_pos0(ptr addrspace(1) %out, float %i
; SDAG-DAZ-NEXT: v_mov_b32_e32 v2, 0x260
; SDAG-DAZ-NEXT: v_cmp_class_f32_e32 vcc, v0, v2
; SDAG-DAZ-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-DAZ-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-DAZ-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SDAG-DAZ-NEXT: s_endpgm
;
@@ -4274,18 +4272,19 @@ entry:
define amdgpu_kernel void @elim_redun_check_ult(ptr addrspace(1) %out, float %in) {
; SDAG-IEEE-LABEL: elim_redun_check_ult:
; SDAG-IEEE: ; %bb.0: ; %entry
-; SDAG-IEEE-NEXT: s_load_dword s0, s[4:5], 0xb
-; SDAG-IEEE-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SDAG-IEEE-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SDAG-IEEE-NEXT: v_mov_b32_e32 v0, 0xf800000
; SDAG-IEEE-NEXT: v_mov_b32_e32 v1, 0x4f800000
-; SDAG-IEEE-NEXT: s_mov_b32 s7, 0xf000
+; SDAG-IEEE-NEXT: s_mov_b32 s11, 0xf000
+; SDAG-IEEE-NEXT: s_mov_b32 s10, -1
; SDAG-IEEE-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-IEEE-NEXT: v_mul_f32_e32 v1, s0, v1
-; SDAG-IEEE-NEXT: v_mov_b32_e32 v2, s0
-; SDAG-IEEE-NEXT: v_cmp_lt_f32_e64 s[0:1], s0, v0
+; SDAG-IEEE-NEXT: v_mul_f32_e32 v1, s6, v1
+; SDAG-IEEE-NEXT: v_mov_b32_e32 v2, s6
+; SDAG-IEEE-NEXT: v_cmp_lt_f32_e64 s[0:1], s6, v0
; SDAG-IEEE-NEXT: v_cndmask_b32_e64 v0, v2, v1, s[0:1]
; SDAG-IEEE-NEXT: v_sqrt_f32_e32 v1, v0
-; SDAG-IEEE-NEXT: s_mov_b32 s6, -1
+; SDAG-IEEE-NEXT: s_mov_b32 s8, s4
+; SDAG-IEEE-NEXT: s_mov_b32 s9, s5
; SDAG-IEEE-NEXT: v_add_i32_e32 v2, vcc, -1, v1
; SDAG-IEEE-NEXT: v_fma_f32 v3, -v2, v1, v0
; SDAG-IEEE-NEXT: v_cmp_ge_f32_e32 vcc, 0, v3
@@ -4299,7 +4298,7 @@ define amdgpu_kernel void @elim_redun_check_ult(ptr addrspace(1) %out, float %in
; SDAG-IEEE-NEXT: v_mov_b32_e32 v2, 0x260
; SDAG-IEEE-NEXT: v_cmp_class_f32_e32 vcc, v0, v2
; SDAG-IEEE-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-IEEE-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SDAG-IEEE-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SDAG-IEEE-NEXT: s_endpgm
;
; GISEL-IEEE-LABEL: elim_redun_check_ult:
@@ -4351,18 +4350,17 @@ define amdgpu_kernel void @elim_redun_check_ult(ptr addrspace(1) %out, float %in
;
; SDAG-DAZ-LABEL: elim_redun_check_ult:
; SDAG-DAZ: ; %bb.0: ; %entry
-; SDAG-DAZ-NEXT: s_load_dword s0, s[4:5], 0xb
+; SDAG-DAZ-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SDAG-DAZ-NEXT: v_mov_b32_e32 v0, 0xf800000
; SDAG-DAZ-NEXT: v_mov_b32_e32 v1, 0x4f800000
-; SDAG-DAZ-NEXT: s_mov_b32 s3, 0xf000
-; SDAG-DAZ-NEXT: s_mov_b32 s2, -1
; SDAG-DAZ-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-DAZ-NEXT: v_mul_f32_e32 v1, s0, v1
-; SDAG-DAZ-NEXT: v_mov_b32_e32 v2, s0
-; SDAG-DAZ-NEXT: v_cmp_lt_f32_e32 vcc, s0, v0
+; SDAG-DAZ-NEXT: s_mov_b32 s3, 0xf000
+; SDAG-DAZ-NEXT: v_mul_f32_e32 v1, s2, v1
+; SDAG-DAZ-NEXT: v_mov_b32_e32 v2, s2
+; SDAG-DAZ-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; SDAG-DAZ-NEXT: v_cndmask_b32_e32 v0, v2, v1, vcc
; SDAG-DAZ-NEXT: v_rsq_f32_e32 v1, v0
-; SDAG-DAZ-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SDAG-DAZ-NEXT: s_mov_b32 s2, -1
; SDAG-DAZ-NEXT: v_mul_f32_e32 v2, v0, v1
; SDAG-DAZ-NEXT: v_mul_f32_e32 v1, 0.5, v1
; SDAG-DAZ-NEXT: v_fma_f32 v3, -v1, v2, 0.5
@@ -4375,7 +4373,6 @@ define amdgpu_kernel void @elim_redun_check_ult(ptr addrspace(1) %out, float %in
; SDAG-DAZ-NEXT: v_mov_b32_e32 v2, 0x260
; SDAG-DAZ-NEXT: v_cmp_class_f32_e32 vcc, v0, v2
; SDAG-DAZ-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-DAZ-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-DAZ-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SDAG-DAZ-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
index 100707a5041d7..dc102c10236aa 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
@@ -15533,16 +15533,12 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
; GFX9-NEXT: v_writelane_b32 v40, s31, 27
; GFX9-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s52, s[34:35], 0x0
-; GFX9-NEXT: ; kill: killed $sgpr34_sgpr35
-; GFX9-NEXT: ; kill: killed $sgpr34_sgpr35
; GFX9-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x40
; GFX9-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0
+; GFX9-NEXT: v_mov_b32_e32 v0, s34
+; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24
; GFX9-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg at abs32@hi
-; GFX9-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg at abs32@lo
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s52
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24
; GFX9-NEXT: v_mov_b32_e32 v0, s46
; GFX9-NEXT: v_mov_b32_e32 v1, s47
; GFX9-NEXT: v_mov_b32_e32 v2, s48
@@ -15554,6 +15550,7 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
; GFX9-NEXT: v_mov_b32_e32 v0, s50
; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:16
; GFX9-NEXT: v_mov_b32_e32 v0, s51
+; GFX9-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg at abs32@lo
; GFX9-NEXT: s_mov_b32 s20, s36
; GFX9-NEXT: s_mov_b32 s21, s37
; GFX9-NEXT: s_mov_b32 s22, s38
@@ -15644,22 +15641,19 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
; GFX10-NEXT: v_writelane_b32 v40, s31, 27
; GFX10-NEXT: s_load_dwordx2 s[34:35], s[34:35], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_clause 0x2
-; GFX10-NEXT: s_load_dword s52, s[34:35], 0x0
-; GFX10-NEXT: ; meta instruction
-; GFX10-NEXT: ; meta instruction
+; GFX10-NEXT: s_clause 0x1
; GFX10-NEXT: s_load_dwordx16 s[36:51], s[34:35], 0x40
; GFX10-NEXT: s_load_dwordx16 s[4:19], s[34:35], 0x0
+; GFX10-NEXT: v_mov_b32_e32 v0, s34
; GFX10-NEXT: s_mov_b32 s35, external_void_func_v32i32_i32_inreg at abs32@hi
; GFX10-NEXT: s_mov_b32 s34, external_void_func_v32i32_i32_inreg at abs32@lo
+; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s52
+; GFX10-NEXT: v_mov_b32_e32 v0, s46
; GFX10-NEXT: v_mov_b32_e32 v1, s47
; GFX10-NEXT: v_mov_b32_e32 v2, s48
; GFX10-NEXT: v_mov_b32_e32 v3, s49
; GFX10-NEXT: s_mov_b32 s20, s36
-; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:24
-; GFX10-NEXT: v_mov_b32_e32 v0, s46
; GFX10-NEXT: s_mov_b32 s21, s37
; GFX10-NEXT: s_mov_b32 s22, s38
; GFX10-NEXT: s_mov_b32 s23, s39
@@ -15755,20 +15749,18 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
; GFX11-NEXT: v_writelane_b32 v40, s30, 26
; GFX11-NEXT: v_writelane_b32 v40, s31, 27
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
-; GFX11-NEXT: s_add_i32 s3, s32, 16
+; GFX11-NEXT: s_add_i32 s2, s32, 16
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b32 s2, s[0:1], 0x0
+; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b512 s[36:51], s[0:1], 0x40
; GFX11-NEXT: s_load_b512 s[4:19], s[0:1], 0x0
+; GFX11-NEXT: v_mov_b32_e32 v6, s0
; GFX11-NEXT: s_mov_b32 s1, external_void_func_v32i32_i32_inreg at abs32@hi
; GFX11-NEXT: s_mov_b32 s0, external_void_func_v32i32_i32_inreg at abs32@lo
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v6, s2 :: v_dual_mov_b32 v5, s51
; GFX11-NEXT: v_dual_mov_b32 v4, s50 :: v_dual_mov_b32 v1, s47
-; GFX11-NEXT: v_dual_mov_b32 v0, s46 :: v_dual_mov_b32 v3, s49
-; GFX11-NEXT: v_mov_b32_e32 v2, s48
-; GFX11-NEXT: s_add_i32 s2, s32, 24
+; GFX11-NEXT: v_dual_mov_b32 v5, s51 :: v_dual_mov_b32 v0, s46
+; GFX11-NEXT: v_dual_mov_b32 v3, s49 :: v_dual_mov_b32 v2, s48
; GFX11-NEXT: s_mov_b32 s20, s36
; GFX11-NEXT: s_mov_b32 s21, s37
; GFX11-NEXT: s_mov_b32 s22, s38
@@ -15779,8 +15771,7 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
; GFX11-NEXT: s_mov_b32 s27, s43
; GFX11-NEXT: s_mov_b32 s28, s44
; GFX11-NEXT: s_mov_b32 s29, s45
-; GFX11-NEXT: scratch_store_b32 off, v6, s2
-; GFX11-NEXT: scratch_store_b64 off, v[4:5], s3
+; GFX11-NEXT: scratch_store_b96 off, v[4:6], s2
; GFX11-NEXT: scratch_store_b128 off, v[0:3], s32
; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-NEXT: v_readlane_b32 s30, v40, 26
@@ -15860,25 +15851,21 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s30, 26
; GFX10-SCRATCH-NEXT: v_writelane_b32 v40, s31, 27
; GFX10-SCRATCH-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX10-SCRATCH-NEXT: s_add_i32 s3, s32, 16
+; GFX10-SCRATCH-NEXT: s_add_i32 s2, s32, 16
; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SCRATCH-NEXT: s_clause 0x2
-; GFX10-SCRATCH-NEXT: s_load_dword s2, s[0:1], 0x0
-; GFX10-SCRATCH-NEXT: ; meta instruction
-; GFX10-SCRATCH-NEXT: ; meta instruction
+; GFX10-SCRATCH-NEXT: s_clause 0x1
; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[36:51], s[0:1], 0x40
; GFX10-SCRATCH-NEXT: s_load_dwordx16 s[4:19], s[0:1], 0x0
+; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v6, s0
; GFX10-SCRATCH-NEXT: s_mov_b32 s1, external_void_func_v32i32_i32_inreg at abs32@hi
; GFX10-SCRATCH-NEXT: s_mov_b32 s0, external_void_func_v32i32_i32_inreg at abs32@lo
; GFX10-SCRATCH-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v6, s2
; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v4, s50
; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v5, s51
; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v0, s46
; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v1, s47
; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v2, s48
; GFX10-SCRATCH-NEXT: v_mov_b32_e32 v3, s49
-; GFX10-SCRATCH-NEXT: s_add_i32 s2, s32, 24
; GFX10-SCRATCH-NEXT: s_mov_b32 s20, s36
; GFX10-SCRATCH-NEXT: s_mov_b32 s21, s37
; GFX10-SCRATCH-NEXT: s_mov_b32 s22, s38
@@ -15889,8 +15876,7 @@ define amdgpu_gfx void @test_call_external_void_func_v32i32_i32_inreg(i32) #0 {
; GFX10-SCRATCH-NEXT: s_mov_b32 s27, s43
; GFX10-SCRATCH-NEXT: s_mov_b32 s28, s44
; GFX10-SCRATCH-NEXT: s_mov_b32 s29, s45
-; GFX10-SCRATCH-NEXT: scratch_store_dword off, v6, s2
-; GFX10-SCRATCH-NEXT: scratch_store_dwordx2 off, v[4:5], s3
+; GFX10-SCRATCH-NEXT: scratch_store_dwordx3 off, v[4:6], s2
; GFX10-SCRATCH-NEXT: scratch_store_dwordx4 off, v[0:3], s32
; GFX10-SCRATCH-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX10-SCRATCH-NEXT: v_readlane_b32 s30, v40, 26
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics.ll b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
index 580eeda73781e..be1fc4f75cc2a 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics.ll
@@ -8,34 +8,35 @@
define amdgpu_kernel void @atomic_add_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_add_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_add v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_add v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_add_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_add v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_add v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_add_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -46,9 +47,7 @@ define amdgpu_kernel void @atomic_add_i32_offset(ptr addrspace(1) %out, i32 %in)
;
; GFX11-LABEL: atomic_add_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:16
@@ -65,23 +64,22 @@ entry:
define amdgpu_kernel void @atomic_add_i32_max_neg_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_add_i32_max_neg_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_mov_b32_e32 v0, 0xfffff000
; SI-NEXT: v_mov_b32_e32 v1, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s6
-; SI-NEXT: buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_add_i32_max_neg_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 0xfffff000
; VI-NEXT: s_addc_u32 s1, s1, -1
@@ -95,8 +93,7 @@ define amdgpu_kernel void @atomic_add_i32_max_neg_offset(ptr addrspace(1) %out,
;
; GFX9-LABEL: atomic_add_i32_max_neg_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -107,9 +104,7 @@ define amdgpu_kernel void @atomic_add_i32_max_neg_offset(ptr addrspace(1) %out,
;
; GFX11-LABEL: atomic_add_i32_max_neg_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:-4096
@@ -126,36 +121,37 @@ entry:
define amdgpu_kernel void @atomic_add_i32_soffset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_add_i32_soffset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b32 s4, 0x8ca0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_add v0, off, s[0:3], s4
+; SI-NEXT: s_mov_b32 s3, 0x8ca0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_add v0, off, s[4:7], s3
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_add_i32_soffset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: s_mov_b32 s4, 0x8ca0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_add v0, off, s[0:3], s4
+; VI-NEXT: s_mov_b32 s3, 0x8ca0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_add v0, off, s[4:7], s3
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_add_i32_soffset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0x8000
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -166,9 +162,7 @@ define amdgpu_kernel void @atomic_add_i32_soffset(ptr addrspace(1) %out, i32 %in
;
; GFX11-LABEL: atomic_add_i32_soffset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0x8000 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:3232
@@ -185,23 +179,22 @@ entry:
define amdgpu_kernel void @atomic_add_i32_huge_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_add_i32_huge_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_mov_b32_e32 v0, 0xdeac
; SI-NEXT: v_mov_b32_e32 v1, 0xabcd
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_add_i32_huge_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 0xdeac
; VI-NEXT: s_addc_u32 s1, s1, 0xabcd
@@ -215,8 +208,7 @@ define amdgpu_kernel void @atomic_add_i32_huge_offset(ptr addrspace(1) %out, i32
;
; GFX9-LABEL: atomic_add_i32_huge_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_add_u32 s0, s0, 0xdeac
@@ -229,9 +221,7 @@ define amdgpu_kernel void @atomic_add_i32_huge_offset(ptr addrspace(1) %out, i32
;
; GFX11-LABEL: atomic_add_i32_huge_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: s_add_u32 s0, s0, 0xdeac
@@ -321,35 +311,34 @@ entry:
define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_add_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_add_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_add v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -357,15 +346,14 @@ define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX9-LABEL: atomic_add_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_add v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -373,16 +361,15 @@ define amdgpu_kernel void @atomic_add_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX11-LABEL: atomic_add_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_add_u32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -488,34 +475,35 @@ entry:
define amdgpu_kernel void @atomic_add_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_add_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_add v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_add v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_add_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_add v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_add v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_add_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -526,9 +514,7 @@ define amdgpu_kernel void @atomic_add_i32(ptr addrspace(1) %out, i32 %in) {
;
; GFX11-LABEL: atomic_add_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
@@ -613,33 +599,32 @@ entry:
define amdgpu_kernel void @atomic_add_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_add_i32_addr64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_add v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_add v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_add_i32_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_add v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -647,15 +632,14 @@ define amdgpu_kernel void @atomic_add_i32_addr64(ptr addrspace(1) %out, i32 %in,
;
; GFX9-LABEL: atomic_add_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_add v0, v1, s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -663,16 +647,15 @@ define amdgpu_kernel void @atomic_add_i32_addr64(ptr addrspace(1) %out, i32 %in,
;
; GFX11-LABEL: atomic_add_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_add_u32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -774,34 +757,35 @@ entry:
define amdgpu_kernel void @atomic_and_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_and_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_and v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_and v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_and_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_and v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_and v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_and_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -812,9 +796,7 @@ define amdgpu_kernel void @atomic_and_i32_offset(ptr addrspace(1) %out, i32 %in)
;
; GFX11-LABEL: atomic_and_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_and_b32 v0, v1, s[0:1] offset:16
@@ -901,35 +883,34 @@ entry:
define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_and_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_and v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_and v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_and_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_and v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -937,15 +918,14 @@ define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX9-LABEL: atomic_and_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_and v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -953,16 +933,15 @@ define amdgpu_kernel void @atomic_and_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX11-LABEL: atomic_and_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_and_b32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -1068,34 +1047,35 @@ entry:
define amdgpu_kernel void @atomic_and_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_and_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_and v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_and v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_and_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_and v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_and v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_and_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -1106,9 +1086,7 @@ define amdgpu_kernel void @atomic_and_i32(ptr addrspace(1) %out, i32 %in) {
;
; GFX11-LABEL: atomic_and_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_and_b32 v0, v1, s[0:1]
@@ -1193,33 +1171,32 @@ entry:
define amdgpu_kernel void @atomic_and_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_and_i32_addr64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_and v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_and v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_and_i32_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_and v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -1227,15 +1204,14 @@ define amdgpu_kernel void @atomic_and_i32_addr64(ptr addrspace(1) %out, i32 %in,
;
; GFX9-LABEL: atomic_and_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_and v0, v1, s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -1243,16 +1219,15 @@ define amdgpu_kernel void @atomic_and_i32_addr64(ptr addrspace(1) %out, i32 %in,
;
; GFX11-LABEL: atomic_and_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_and_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -1354,34 +1329,35 @@ entry:
define amdgpu_kernel void @atomic_sub_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_sub_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_sub v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_sub v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_sub_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_sub v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_sub v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_sub_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -1392,9 +1368,7 @@ define amdgpu_kernel void @atomic_sub_i32_offset(ptr addrspace(1) %out, i32 %in)
;
; GFX11-LABEL: atomic_sub_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_sub_u32 v0, v1, s[0:1] offset:16
@@ -1481,35 +1455,34 @@ entry:
define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_sub_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_sub v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_sub v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_sub_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_sub v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -1517,15 +1490,14 @@ define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX9-LABEL: atomic_sub_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_sub v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -1533,16 +1505,15 @@ define amdgpu_kernel void @atomic_sub_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX11-LABEL: atomic_sub_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_sub_u32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -1648,34 +1619,35 @@ entry:
define amdgpu_kernel void @atomic_sub_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_sub_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_sub v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_sub v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_sub_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_sub v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_sub v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_sub_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -1686,9 +1658,7 @@ define amdgpu_kernel void @atomic_sub_i32(ptr addrspace(1) %out, i32 %in) {
;
; GFX11-LABEL: atomic_sub_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_sub_u32 v0, v1, s[0:1]
@@ -1773,33 +1743,32 @@ entry:
define amdgpu_kernel void @atomic_sub_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_sub_i32_addr64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_sub v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_sub v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_sub_i32_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_sub v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -1807,15 +1776,14 @@ define amdgpu_kernel void @atomic_sub_i32_addr64(ptr addrspace(1) %out, i32 %in,
;
; GFX9-LABEL: atomic_sub_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_sub v0, v1, s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -1823,16 +1791,15 @@ define amdgpu_kernel void @atomic_sub_i32_addr64(ptr addrspace(1) %out, i32 %in,
;
; GFX11-LABEL: atomic_sub_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_sub_u32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -1934,34 +1901,35 @@ entry:
define amdgpu_kernel void @atomic_max_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_max_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_smax v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_smax v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_max_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_smax v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_smax v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -1972,9 +1940,7 @@ define amdgpu_kernel void @atomic_max_i32_offset(ptr addrspace(1) %out, i32 %in)
;
; GFX11-LABEL: atomic_max_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_max_i32 v0, v1, s[0:1] offset:16
@@ -2057,62 +2023,59 @@ entry:
define amdgpu_kernel void @atomic_max_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_max_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_smax v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_smax v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_max_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_smax v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_smax v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_max_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_max_i32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
@@ -2213,30 +2176,31 @@ entry:
define amdgpu_kernel void @atomic_max_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_max_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_smax v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_smax v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_max_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_smax v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_smax v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -2245,9 +2209,7 @@ define amdgpu_kernel void @atomic_max_i32(ptr addrspace(1) %out, i32 %in) {
;
; GFX11-LABEL: atomic_max_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_max_i32 v0, v1, s[0:1]
@@ -2327,60 +2289,57 @@ entry:
define amdgpu_kernel void @atomic_max_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_max_i32_addr64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_smax v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_smax v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_max_i32_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_smax v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_max_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_smax v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_max_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_max_i32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
@@ -2477,30 +2436,31 @@ entry:
define amdgpu_kernel void @atomic_umax_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_umax_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_umax v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_umax v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_umax_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_umax v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_umax v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -2509,9 +2469,7 @@ define amdgpu_kernel void @atomic_umax_i32_offset(ptr addrspace(1) %out, i32 %in
;
; GFX11-LABEL: atomic_umax_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_max_u32 v0, v1, s[0:1] offset:16
@@ -2593,62 +2551,59 @@ entry:
define amdgpu_kernel void @atomic_umax_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_umax_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_umax v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_umax v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_umax_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_umax v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_umax v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_umax_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_max_u32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
@@ -2749,30 +2704,31 @@ entry:
define amdgpu_kernel void @atomic_umax_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_umax_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_umax v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_umax v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_umax_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_umax v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_umax v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -2781,9 +2737,7 @@ define amdgpu_kernel void @atomic_umax_i32(ptr addrspace(1) %out, i32 %in) {
;
; GFX11-LABEL: atomic_umax_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_max_u32 v0, v1, s[0:1]
@@ -2863,60 +2817,57 @@ entry:
define amdgpu_kernel void @atomic_umax_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_umax_i32_addr64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_umax v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_umax v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_umax_i32_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_umax v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umax_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_umax v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_umax_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_max_u32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
@@ -3013,30 +2964,31 @@ entry:
define amdgpu_kernel void @atomic_min_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_min_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_smin v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_smin v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_min_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_smin v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_smin v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -3045,9 +2997,7 @@ define amdgpu_kernel void @atomic_min_i32_offset(ptr addrspace(1) %out, i32 %in)
;
; GFX11-LABEL: atomic_min_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_min_i32 v0, v1, s[0:1] offset:16
@@ -3129,62 +3079,59 @@ entry:
define amdgpu_kernel void @atomic_min_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_min_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_smin v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_smin v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_min_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_smin v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_smin v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_min_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_min_i32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
@@ -3285,30 +3232,31 @@ entry:
define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_min_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_smin v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_smin v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_min_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_smin v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_smin v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -3317,9 +3265,7 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
;
; GFX11-LABEL: atomic_min_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_min_i32 v0, v1, s[0:1]
@@ -3399,60 +3345,57 @@ entry:
define amdgpu_kernel void @atomic_min_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_min_i32_addr64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_smin v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_smin v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_min_i32_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_smin v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_smin v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_min_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_min_i32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
@@ -3549,30 +3492,31 @@ entry:
define amdgpu_kernel void @atomic_umin_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_umin_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_umin v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_umin v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_umin_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_umin v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_umin v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umin_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -3581,9 +3525,7 @@ define amdgpu_kernel void @atomic_umin_i32_offset(ptr addrspace(1) %out, i32 %in
;
; GFX11-LABEL: atomic_umin_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_min_u32 v0, v1, s[0:1] offset:16
@@ -3665,62 +3607,59 @@ entry:
define amdgpu_kernel void @atomic_umin_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_umin_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_umin v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_umin v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_umin_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_umin v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umin_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_umin v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_umin_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_min_u32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
@@ -3821,30 +3760,31 @@ entry:
define amdgpu_kernel void @atomic_umin_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_umin_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_umin v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_umin v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_umin_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_umin v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_umin v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umin_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -3853,9 +3793,7 @@ define amdgpu_kernel void @atomic_umin_i32(ptr addrspace(1) %out, i32 %in) {
;
; GFX11-LABEL: atomic_umin_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_min_u32 v0, v1, s[0:1]
@@ -3935,60 +3873,57 @@ entry:
define amdgpu_kernel void @atomic_umin_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_umin_i32_addr64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_umin v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_umin v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_umin_i32_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_umin v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_umin_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_umin v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_umin_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_min_u32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl0_inv
@@ -4085,34 +4020,35 @@ entry:
define amdgpu_kernel void @atomic_or_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_or_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_or v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_or v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_or_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_or v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_or v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_or_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -4123,9 +4059,7 @@ define amdgpu_kernel void @atomic_or_i32_offset(ptr addrspace(1) %out, i32 %in)
;
; GFX11-LABEL: atomic_or_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_or_b32 v0, v1, s[0:1] offset:16
@@ -4212,35 +4146,34 @@ entry:
define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_or_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_or v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_or v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_or_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_or v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -4248,15 +4181,14 @@ define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr addrspace(1) %out, i3
;
; GFX9-LABEL: atomic_or_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_or v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -4264,16 +4196,15 @@ define amdgpu_kernel void @atomic_or_i32_addr64_offset(ptr addrspace(1) %out, i3
;
; GFX11-LABEL: atomic_or_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_or_b32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -4379,34 +4310,35 @@ entry:
define amdgpu_kernel void @atomic_or_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_or_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_or v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_or v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_or_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_or v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_or v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_or_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -4417,9 +4349,7 @@ define amdgpu_kernel void @atomic_or_i32(ptr addrspace(1) %out, i32 %in) {
;
; GFX11-LABEL: atomic_or_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_or_b32 v0, v1, s[0:1]
@@ -4504,33 +4434,32 @@ entry:
define amdgpu_kernel void @atomic_or_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_or_i32_addr64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_or v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_or v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_or_i32_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_or v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -4538,15 +4467,14 @@ define amdgpu_kernel void @atomic_or_i32_addr64(ptr addrspace(1) %out, i32 %in,
;
; GFX9-LABEL: atomic_or_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_or v0, v1, s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -4554,16 +4482,15 @@ define amdgpu_kernel void @atomic_or_i32_addr64(ptr addrspace(1) %out, i32 %in,
;
; GFX11-LABEL: atomic_or_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_or_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -4665,34 +4592,35 @@ entry:
define amdgpu_kernel void @atomic_xchg_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_xchg_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_swap v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_swap v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_xchg_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_swap v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_swap v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_xchg_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -4703,9 +4631,7 @@ define amdgpu_kernel void @atomic_xchg_i32_offset(ptr addrspace(1) %out, i32 %in
;
; GFX11-LABEL: atomic_xchg_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] offset:16
@@ -4722,34 +4648,35 @@ entry:
define amdgpu_kernel void @atomic_xchg_f32_offset(ptr addrspace(1) %out, float %in) {
; SI-LABEL: atomic_xchg_f32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_swap v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_swap v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_xchg_f32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_swap v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_swap v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_xchg_f32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -4760,9 +4687,7 @@ define amdgpu_kernel void @atomic_xchg_f32_offset(ptr addrspace(1) %out, float %
;
; GFX11-LABEL: atomic_xchg_f32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] offset:16
@@ -4849,35 +4774,34 @@ entry:
define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_xchg_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_swap v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_xchg_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_swap v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -4885,15 +4809,14 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(ptr addrspace(1) %out,
;
; GFX9-LABEL: atomic_xchg_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_swap v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -4901,16 +4824,15 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64_offset(ptr addrspace(1) %out,
;
; GFX11-LABEL: atomic_xchg_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_swap_b32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -5016,34 +4938,35 @@ entry:
define amdgpu_kernel void @atomic_xchg_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_xchg_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_xchg_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_swap v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_swap v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_xchg_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -5054,9 +4977,7 @@ define amdgpu_kernel void @atomic_xchg_i32(ptr addrspace(1) %out, i32 %in) {
;
; GFX11-LABEL: atomic_xchg_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
@@ -5141,33 +5062,32 @@ entry:
define amdgpu_kernel void @atomic_xchg_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_xchg_i32_addr64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_swap v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_swap v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_xchg_i32_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_swap v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -5175,15 +5095,14 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64(ptr addrspace(1) %out, i32 %in
;
; GFX9-LABEL: atomic_xchg_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_swap v0, v1, s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -5191,16 +5110,15 @@ define amdgpu_kernel void @atomic_xchg_i32_addr64(ptr addrspace(1) %out, i32 %in
;
; GFX11-LABEL: atomic_xchg_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_swap_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -5321,9 +5239,9 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr addrspace(1) %out, i32
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -5345,9 +5263,9 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_offset(ptr addrspace(1) %out, i32
; GFX11-LABEL: atomic_cmpxchg_i32_offset:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -5387,10 +5305,10 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_offset(ptr addrspace(1) %out,
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 s4, s2
-; VI-NEXT: v_mov_b32_e32 v0, s8
; VI-NEXT: s_mov_b32 s5, s3
; VI-NEXT: s_mov_b32 s2, s6
; VI-NEXT: s_mov_b32 s3, s7
+; VI-NEXT: v_mov_b32_e32 v0, s8
; VI-NEXT: v_mov_b32_e32 v1, s9
; VI-NEXT: buffer_atomic_cmpswap v[0:1], off, s[0:3], 0 offset:16 glc
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -5417,9 +5335,9 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_offset(ptr addrspace(1) %out,
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s7
+; GFX11-NEXT: v_mov_b32_e32 v0, s6
; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
@@ -5437,38 +5355,33 @@ entry:
define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index, i32 %old) {
; SI-LABEL: atomic_cmpxchg_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s9, s[4:5], 0xf
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v0, s8
-; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; SI-NEXT: s_mov_b32 s10, 0
+; SI-NEXT: s_mov_b64 s[8:9], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: v_mov_b32_e32 v3, s5
-; SI-NEXT: buffer_atomic_cmpswap v[0:1], v[2:3], s[0:3], 0 addr64 offset:16
+; SI-NEXT: buffer_atomic_cmpswap v[0:1], v[2:3], s[8:11], 0 addr64 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_cmpxchg_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x3c
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s6
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -5477,17 +5390,14 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(ptr addrspace(1) %ou
;
; GFX9-LABEL: atomic_cmpxchg_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x3c
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[12:13], 2
+; GFX9-NEXT: s_add_u32 s0, s8, s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-NEXT: s_addc_u32 s1, s9, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s14
; GFX9-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -5495,17 +5405,13 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64_offset(ptr addrspace(1) %ou
;
; GFX11-LABEL: atomic_cmpxchg_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x3
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b32 s7, s[4:5], 0x3c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -5522,9 +5428,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64_offset(ptr addrspace(1)
; SI-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xf
; SI-NEXT: s_load_dword s6, s[4:5], 0xd
-; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xf
-; SI-NEXT: s_load_dword s10, s[4:5], 0x11
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s4, s2
@@ -5545,67 +5450,67 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64_offset(ptr addrspace(1)
;
; VI-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; VI-NEXT: s_load_dword s8, s[4:5], 0x34
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s9, s[4:5], 0x44
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x3c
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
+; VI-NEXT: s_load_dword s3, s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_add_u32 s0, s4, s0
+; VI-NEXT: s_addc_u32 s1, s5, s1
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v0, s3
+; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: s_mov_b32 s7, 0xf000
-; VI-NEXT: s_mov_b32 s6, -1
-; VI-NEXT: s_mov_b32 s4, s2
-; VI-NEXT: s_mov_b32 s5, s3
-; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s0, s6
+; VI-NEXT: s_mov_b32 s1, s7
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX9-NEXT: s_load_dword s9, s[4:5], 0x44
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x3c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s3, s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; GFX9-NEXT: s_add_u32 s0, s0, s4
-; GFX9-NEXT: s_addc_u32 s1, s1, s5
-; GFX9-NEXT: v_mov_b32_e32 v0, s8
-; GFX9-NEXT: v_mov_b32_e32 v1, s9
+; GFX9-NEXT: s_add_u32 s0, s8, s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: s_addc_u32 s1, s9, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] offset:16 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: global_store_dword v2, v0, s[2:3]
+; GFX9-NEXT: global_store_dword v2, v0, s[10:11]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x3
-; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
-; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s9, s[4:5], 0x44
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x3c
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
-; GFX11-NEXT: s_add_u32 s0, s0, s4
-; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[4:7], s[4:5], 0x24
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v0, s3
+; GFX11-NEXT: s_add_u32 s0, s4, s0
+; GFX11-NEXT: s_addc_u32 s1, s5, s1
; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] offset:16 glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: global_store_b32 v2, v0, s[2:3]
+; GFX11-NEXT: global_store_b32 v2, v0, s[6:7]
; GFX11-NEXT: s_endpgm
entry:
%ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
@@ -5638,9 +5543,9 @@ define amdgpu_kernel void @atomic_cmpxchg_i32(ptr addrspace(1) %out, i32 %in, i3
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -5662,9 +5567,9 @@ define amdgpu_kernel void @atomic_cmpxchg_i32(ptr addrspace(1) %out, i32 %in, i3
; GFX11-LABEL: atomic_cmpxchg_i32:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -5703,8 +5608,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret(ptr addrspace(1) %out, ptr add
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 s4, s0
-; VI-NEXT: v_mov_b32_e32 v0, s8
; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s8
; VI-NEXT: v_mov_b32_e32 v1, s9
; VI-NEXT: buffer_atomic_cmpswap v[0:1], off, s[4:7], 0 glc
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -5733,9 +5638,9 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret(ptr addrspace(1) %out, ptr add
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s7
+; GFX11-NEXT: v_mov_b32_e32 v0, s6
; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
@@ -5752,36 +5657,31 @@ entry:
define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index, i32 %old) {
; SI-LABEL: atomic_cmpxchg_i32_addr64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s9, s[4:5], 0xf
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s11, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v0, s8
-; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; SI-NEXT: s_mov_b32 s10, 0
+; SI-NEXT: s_mov_b64 s[8:9], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s6
; SI-NEXT: v_mov_b32_e32 v2, s4
; SI-NEXT: v_mov_b32_e32 v3, s5
-; SI-NEXT: buffer_atomic_cmpswap v[0:1], v[2:3], s[0:3], 0 addr64
+; SI-NEXT: buffer_atomic_cmpswap v[0:1], v[2:3], s[8:11], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_cmpxchg_i32_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x3c
+; VI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s6
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_atomic_cmpswap v[2:3], v[0:1]
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -5790,17 +5690,14 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(ptr addrspace(1) %out, i32
;
; GFX9-LABEL: atomic_cmpxchg_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dword s7, s[4:5], 0x3c
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: v_mov_b32_e32 v1, s7
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[12:13], 2
+; GFX9-NEXT: s_add_u32 s0, s8, s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s10
+; GFX9-NEXT: s_addc_u32 s1, s9, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s14
; GFX9-NEXT: global_atomic_cmpswap v2, v[0:1], s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -5808,17 +5705,13 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_addr64(ptr addrspace(1) %out, i32
;
; GFX11-LABEL: atomic_cmpxchg_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x3
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b32 s7, s[4:5], 0x3c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s7
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s6
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_cmpswap_b32 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -5834,9 +5727,8 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64(ptr addrspace(1) %out,
; SI-LABEL: atomic_cmpxchg_i32_ret_addr64:
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0xf
; SI-NEXT: s_load_dword s6, s[4:5], 0xd
-; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xf
-; SI-NEXT: s_load_dword s10, s[4:5], 0x11
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s4, s2
@@ -5857,65 +5749,65 @@ define amdgpu_kernel void @atomic_cmpxchg_i32_ret_addr64(ptr addrspace(1) %out,
;
; VI-LABEL: atomic_cmpxchg_i32_ret_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; VI-NEXT: s_load_dword s8, s[4:5], 0x34
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s9, s[4:5], 0x44
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x3c
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: s_add_u32 s0, s0, s4
-; VI-NEXT: s_addc_u32 s1, s1, s5
+; VI-NEXT: s_load_dword s3, s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
+; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_add_u32 s0, s4, s0
+; VI-NEXT: s_addc_u32 s1, s5, s1
; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v1, s9
+; VI-NEXT: v_mov_b32_e32 v0, s3
+; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_atomic_cmpswap v0, v[2:3], v[0:1] glc
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
-; VI-NEXT: s_mov_b32 s7, 0xf000
-; VI-NEXT: s_mov_b32 s6, -1
-; VI-NEXT: s_mov_b32 s4, s2
-; VI-NEXT: s_mov_b32 s5, s3
-; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_mov_b32 s0, s6
+; VI-NEXT: s_mov_b32 s1, s7
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_cmpxchg_i32_ret_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s8, s[4:5], 0x34
-; GFX9-NEXT: s_load_dword s9, s[4:5], 0x44
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x3c
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_load_dword s3, s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; GFX9-NEXT: s_add_u32 s0, s0, s4
-; GFX9-NEXT: s_addc_u32 s1, s1, s5
-; GFX9-NEXT: v_mov_b32_e32 v0, s8
-; GFX9-NEXT: v_mov_b32_e32 v1, s9
+; GFX9-NEXT: s_add_u32 s0, s8, s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-NEXT: s_addc_u32 s1, s9, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
-; GFX9-NEXT: global_store_dword v2, v0, s[2:3]
+; GFX9-NEXT: global_store_dword v2, v0, s[10:11]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_cmpxchg_i32_ret_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x3
-; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x3c
-; GFX11-NEXT: s_load_b32 s8, s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s9, s[4:5], 0x44
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x3c
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; GFX11-NEXT: v_dual_mov_b32 v0, s8 :: v_dual_mov_b32 v1, s9
-; GFX11-NEXT: s_add_u32 s0, s0, s4
-; GFX11-NEXT: s_addc_u32 s1, s1, s5
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[4:7], s[4:5], 0x24
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v0, s3
+; GFX11-NEXT: s_add_u32 s0, s4, s0
+; GFX11-NEXT: s_addc_u32 s1, s5, s1
; GFX11-NEXT: global_atomic_cmpswap_b32 v0, v2, v[0:1], s[0:1] glc
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: buffer_gl1_inv
; GFX11-NEXT: buffer_gl0_inv
-; GFX11-NEXT: global_store_b32 v2, v0, s[2:3]
+; GFX11-NEXT: global_store_b32 v2, v0, s[6:7]
; GFX11-NEXT: s_endpgm
entry:
%ptr = getelementptr i32, ptr addrspace(1) %out, i64 %index
@@ -5928,34 +5820,35 @@ entry:
define amdgpu_kernel void @atomic_xor_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_xor_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_xor v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_xor v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_xor_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_xor v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_xor v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_xor_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -5966,9 +5859,7 @@ define amdgpu_kernel void @atomic_xor_i32_offset(ptr addrspace(1) %out, i32 %in)
;
; GFX11-LABEL: atomic_xor_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_xor_b32 v0, v1, s[0:1] offset:16
@@ -6055,35 +5946,34 @@ entry:
define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_xor_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_xor v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_xor v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_xor_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_xor v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -6091,15 +5981,14 @@ define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX9-LABEL: atomic_xor_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_xor v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -6107,16 +5996,15 @@ define amdgpu_kernel void @atomic_xor_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX11-LABEL: atomic_xor_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_xor_b32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -6222,34 +6110,35 @@ entry:
define amdgpu_kernel void @atomic_xor_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_xor_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_xor v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_xor v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_xor_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_xor v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_xor v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_xor_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -6260,9 +6149,7 @@ define amdgpu_kernel void @atomic_xor_i32(ptr addrspace(1) %out, i32 %in) {
;
; GFX11-LABEL: atomic_xor_i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_xor_b32 v0, v1, s[0:1]
@@ -6347,33 +6234,32 @@ entry:
define amdgpu_kernel void @atomic_xor_i32_addr64(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_xor_i32_addr64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_xor v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_xor v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_xor_i32_addr64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_xor v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -6381,15 +6267,14 @@ define amdgpu_kernel void @atomic_xor_i32_addr64(ptr addrspace(1) %out, i32 %in,
;
; GFX9-LABEL: atomic_xor_i32_addr64:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_xor v0, v1, s[0:1]
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -6397,16 +6282,15 @@ define amdgpu_kernel void @atomic_xor_i32_addr64(ptr addrspace(1) %out, i32 %in,
;
; GFX11-LABEL: atomic_xor_i32_addr64:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_xor_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -8183,34 +8067,35 @@ define amdgpu_kernel void @atomic_store_bf16(bfloat %in, ptr addrspace(1) %out)
define amdgpu_kernel void @atomic_inc_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_inc_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_inc v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_inc v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_inc_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_inc v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_inc v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_inc_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -8221,9 +8106,7 @@ define amdgpu_kernel void @atomic_inc_i32_offset(ptr addrspace(1) %out, i32 %in)
;
; GFX11-LABEL: atomic_inc_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[0:1] offset:16
@@ -8240,23 +8123,22 @@ entry:
define amdgpu_kernel void @atomic_inc_i32_max_neg_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_inc_i32_max_neg_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_mov_b32_e32 v0, 0xfffff000
; SI-NEXT: v_mov_b32_e32 v1, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s6
-; SI-NEXT: buffer_atomic_inc v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: buffer_atomic_inc v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_inc_i32_max_neg_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 0xfffff000
; VI-NEXT: s_addc_u32 s1, s1, -1
@@ -8270,8 +8152,7 @@ define amdgpu_kernel void @atomic_inc_i32_max_neg_offset(ptr addrspace(1) %out,
;
; GFX9-LABEL: atomic_inc_i32_max_neg_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -8282,9 +8163,7 @@ define amdgpu_kernel void @atomic_inc_i32_max_neg_offset(ptr addrspace(1) %out,
;
; GFX11-LABEL: atomic_inc_i32_max_neg_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[0:1] offset:-4096
@@ -8301,36 +8180,37 @@ entry:
define amdgpu_kernel void @atomic_inc_i32_soffset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_inc_i32_soffset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b32 s4, 0x8ca0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_inc v0, off, s[0:3], s4
+; SI-NEXT: s_mov_b32 s3, 0x8ca0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_inc v0, off, s[4:7], s3
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_inc_i32_soffset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: s_mov_b32 s4, 0x8ca0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_inc v0, off, s[0:3], s4
+; VI-NEXT: s_mov_b32 s3, 0x8ca0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_inc v0, off, s[4:7], s3
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_inc_i32_soffset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0x8000
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -8341,9 +8221,7 @@ define amdgpu_kernel void @atomic_inc_i32_soffset(ptr addrspace(1) %out, i32 %in
;
; GFX11-LABEL: atomic_inc_i32_soffset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0x8000 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[0:1] offset:3232
@@ -8360,23 +8238,22 @@ entry:
define amdgpu_kernel void @atomic_inc_i32_huge_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_inc_i32_huge_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_mov_b32_e32 v0, 0xdeac
; SI-NEXT: v_mov_b32_e32 v1, 0xabcd
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_atomic_inc v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: buffer_atomic_inc v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_inc_i32_huge_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 0xdeac
; VI-NEXT: s_addc_u32 s1, s1, 0xabcd
@@ -8390,8 +8267,7 @@ define amdgpu_kernel void @atomic_inc_i32_huge_offset(ptr addrspace(1) %out, i32
;
; GFX9-LABEL: atomic_inc_i32_huge_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_add_u32 s0, s0, 0xdeac
@@ -8404,9 +8280,7 @@ define amdgpu_kernel void @atomic_inc_i32_huge_offset(ptr addrspace(1) %out, i32
;
; GFX11-LABEL: atomic_inc_i32_huge_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: s_add_u32 s0, s0, 0xdeac
@@ -8495,35 +8369,34 @@ entry:
define amdgpu_kernel void @atomic_inc_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_inc_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_inc v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_inc v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_inc_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_inc v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -8531,15 +8404,14 @@ define amdgpu_kernel void @atomic_inc_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX9-LABEL: atomic_inc_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_inc v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -8547,16 +8419,15 @@ define amdgpu_kernel void @atomic_inc_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX11-LABEL: atomic_inc_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_inc_u32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -8662,34 +8533,35 @@ entry:
define amdgpu_kernel void @atomic_dec_i32_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_dec_i32_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_dec v0, off, s[0:3], 0 offset:16
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_dec v0, off, s[4:7], 0 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_dec_i32_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_dec v0, off, s[0:3], 0 offset:16
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_dec v0, off, s[4:7], 0 offset:16
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_dec_i32_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -8700,9 +8572,7 @@ define amdgpu_kernel void @atomic_dec_i32_offset(ptr addrspace(1) %out, i32 %in)
;
; GFX11-LABEL: atomic_dec_i32_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_dec_u32 v0, v1, s[0:1] offset:16
@@ -8719,23 +8589,22 @@ entry:
define amdgpu_kernel void @atomic_dec_i32_max_neg_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_dec_i32_max_neg_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_mov_b32_e32 v0, 0xfffff000
; SI-NEXT: v_mov_b32_e32 v1, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s6
-; SI-NEXT: buffer_atomic_dec v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: buffer_atomic_dec v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_dec_i32_max_neg_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 0xfffff000
; VI-NEXT: s_addc_u32 s1, s1, -1
@@ -8749,8 +8618,7 @@ define amdgpu_kernel void @atomic_dec_i32_max_neg_offset(ptr addrspace(1) %out,
;
; GFX9-LABEL: atomic_dec_i32_max_neg_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -8761,9 +8629,7 @@ define amdgpu_kernel void @atomic_dec_i32_max_neg_offset(ptr addrspace(1) %out,
;
; GFX11-LABEL: atomic_dec_i32_max_neg_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_dec_u32 v0, v1, s[0:1] offset:-4096
@@ -8780,36 +8646,37 @@ entry:
define amdgpu_kernel void @atomic_dec_i32_soffset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_dec_i32_soffset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_mov_b32 s4, 0x8ca0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_atomic_dec v0, off, s[0:3], s4
+; SI-NEXT: s_mov_b32 s3, 0x8ca0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_atomic_dec v0, off, s[4:7], s3
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_dec_i32_soffset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: s_mov_b32 s4, 0x8ca0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_atomic_dec v0, off, s[0:3], s4
+; VI-NEXT: s_mov_b32 s3, 0x8ca0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_atomic_dec v0, off, s[4:7], s3
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_dec_i32_soffset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0x8000
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -8820,9 +8687,7 @@ define amdgpu_kernel void @atomic_dec_i32_soffset(ptr addrspace(1) %out, i32 %in
;
; GFX11-LABEL: atomic_dec_i32_soffset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0x8000 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_atomic_dec_u32 v0, v1, s[0:1] offset:3232
@@ -8839,23 +8704,22 @@ entry:
define amdgpu_kernel void @atomic_dec_i32_huge_offset(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_dec_i32_huge_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_mov_b32_e32 v0, 0xdeac
; SI-NEXT: v_mov_b32_e32 v1, 0xabcd
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: buffer_atomic_dec v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: buffer_atomic_dec v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_dec_i32_huge_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s0, s0, 0xdeac
; VI-NEXT: s_addc_u32 s1, s1, 0xabcd
@@ -8869,8 +8733,7 @@ define amdgpu_kernel void @atomic_dec_i32_huge_offset(ptr addrspace(1) %out, i32
;
; GFX9-LABEL: atomic_dec_i32_huge_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_add_u32 s0, s0, 0xdeac
@@ -8883,9 +8746,7 @@ define amdgpu_kernel void @atomic_dec_i32_huge_offset(ptr addrspace(1) %out, i32
;
; GFX11-LABEL: atomic_dec_i32_huge_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: s_add_u32 s0, s0, 0xdeac
@@ -8974,35 +8835,34 @@ entry:
define amdgpu_kernel void @atomic_dec_i32_addr64_offset(ptr addrspace(1) %out, i32 %in, i64 %index) {
; SI-LABEL: atomic_dec_i32_addr64_offset:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
-; SI-NEXT: s_mov_b32 s2, 0
-; SI-NEXT: v_mov_b32_e32 v2, s8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_atomic_dec v2, v[0:1], s[0:3], 0 addr64 offset:16
+; SI-NEXT: s_lshl_b64 s[8:9], s[8:9], 2
+; SI-NEXT: s_mov_b32 s6, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s8
+; SI-NEXT: v_mov_b32_e32 v1, s9
+; SI-NEXT: buffer_atomic_dec v2, v[0:1], s[4:7], 0 addr64 offset:16
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_dec_i32_addr64_offset:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; VI-NEXT: s_load_dword s4, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; VI-NEXT: s_add_u32 s0, s2, s0
-; VI-NEXT: s_addc_u32 s1, s3, s1
+; VI-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; VI-NEXT: s_add_u32 s0, s0, s4
+; VI-NEXT: s_addc_u32 s1, s1, s5
; VI-NEXT: s_add_u32 s0, s0, 16
; VI-NEXT: s_addc_u32 s1, s1, 0
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: flat_atomic_dec v[0:1], v2
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
@@ -9010,15 +8870,14 @@ define amdgpu_kernel void @atomic_dec_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX9-LABEL: atomic_dec_i32_addr64_offset:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX9-NEXT: s_add_u32 s0, s2, s0
-; GFX9-NEXT: s_addc_u32 s1, s3, s1
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX9-NEXT: s_add_u32 s0, s0, s4
+; GFX9-NEXT: s_addc_u32 s1, s1, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: global_atomic_dec v0, v1, s[0:1] offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
@@ -9026,16 +8885,15 @@ define amdgpu_kernel void @atomic_dec_i32_addr64_offset(ptr addrspace(1) %out, i
;
; GFX11-LABEL: atomic_dec_i32_addr64_offset:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x2
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x24
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s6
-; GFX11-NEXT: s_add_u32 s0, s2, s0
-; GFX11-NEXT: s_addc_u32 s1, s3, s1
+; GFX11-NEXT: s_lshl_b64 s[4:5], s[6:7], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
+; GFX11-NEXT: s_add_u32 s0, s0, s4
+; GFX11-NEXT: s_addc_u32 s1, s1, s5
; GFX11-NEXT: global_atomic_dec_u32 v0, v1, s[0:1] offset:16
; GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-NEXT: buffer_gl1_inv
@@ -9441,11 +9299,10 @@ define amdgpu_kernel void @atomic_load_bf16_negoffset(ptr addrspace(1) %in, ptr
define amdgpu_kernel void @atomic_sub_i16_soffset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i16 %in) {
; SI-LABEL: atomic_sub_i16_soffset__amdgpu_no_remote_memory:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_add_u32 s6, s0, 0x4650
; SI-NEXT: s_addc_u32 s1, s1, 0
; SI-NEXT: s_and_b32 s0, s6, -4
@@ -9481,8 +9338,7 @@ define amdgpu_kernel void @atomic_sub_i16_soffset__amdgpu_no_remote_memory(ptr a
;
; VI-LABEL: atomic_sub_i16_soffset__amdgpu_no_remote_memory:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s3, s0, 0x4650
@@ -9491,18 +9347,18 @@ define amdgpu_kernel void @atomic_sub_i16_soffset__amdgpu_no_remote_memory(ptr a
; VI-NEXT: s_load_dword s9, s[0:1], 0x0
; VI-NEXT: s_and_b32 s3, s3, 3
; VI-NEXT: s_lshl_b32 s3, s3, 3
-; VI-NEXT: s_lshl_b32 s6, 0xffff, s3
; VI-NEXT: s_and_b32 s2, s2, 0xffff
-; VI-NEXT: s_not_b32 s7, s6
-; VI-NEXT: s_lshl_b32 s8, s2, s3
+; VI-NEXT: s_lshl_b32 s6, 0xffff, s3
+; VI-NEXT: s_lshl_b32 s7, s2, s3
+; VI-NEXT: s_not_b32 s8, s6
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s9
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: .LBB136_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_subrev_u32_e32 v0, vcc, s8, v1
-; VI-NEXT: v_and_b32_e32 v2, s7, v1
+; VI-NEXT: v_subrev_u32_e32 v0, vcc, s7, v1
+; VI-NEXT: v_and_b32_e32 v2, s8, v1
; VI-NEXT: v_and_b32_e32 v0, s6, v0
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: v_mov_b32_e32 v3, v1
@@ -9520,44 +9376,41 @@ define amdgpu_kernel void @atomic_sub_i16_soffset__amdgpu_no_remote_memory(ptr a
;
; GFX9-LABEL: atomic_sub_i16_soffset__amdgpu_no_remote_memory:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_add_u32 s3, s0, 0x4650
-; GFX9-NEXT: s_addc_u32 s1, s1, 0
-; GFX9-NEXT: s_and_b32 s0, s3, -4
-; GFX9-NEXT: s_load_dword s7, s[0:1], 0x0
-; GFX9-NEXT: s_and_b32 s3, s3, 3
-; GFX9-NEXT: s_lshl_b32 s3, s3, 3
-; GFX9-NEXT: s_lshl_b32 s4, 0xffff, s3
-; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX9-NEXT: s_not_b32 s5, s4
-; GFX9-NEXT: s_lshl_b32 s6, s2, s3
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_add_u32 s4, s8, 0x4650
+; GFX9-NEXT: s_addc_u32 s3, s9, 0
+; GFX9-NEXT: s_and_b32 s2, s4, -4
+; GFX9-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX9-NEXT: s_and_b32 s4, s4, 3
+; GFX9-NEXT: s_lshl_b32 s6, s4, 3
+; GFX9-NEXT: s_and_b32 s5, s10, 0xffff
+; GFX9-NEXT: s_lshl_b32 s4, 0xffff, s6
+; GFX9-NEXT: s_lshl_b32 s5, s5, s6
+; GFX9-NEXT: s_not_b32 s6, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s7
; GFX9-NEXT: .LBB136_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_subrev_u32_e32 v0, s6, v1
+; GFX9-NEXT: v_subrev_u32_e32 v0, s5, v1
; GFX9-NEXT: v_and_b32_e32 v0, s4, v0
-; GFX9-NEXT: v_and_or_b32 v0, v1, s5, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB136_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_sub_i16_soffset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_add_u32 s3, s0, 0x4650
@@ -9601,11 +9454,10 @@ define amdgpu_kernel void @atomic_sub_i16_soffset__amdgpu_no_remote_memory(ptr a
define amdgpu_kernel void @atomic_sub_i8_soffset__amdgpu_no_remote_memory(ptr addrspace(1) %out, i8 %in) {
; SI-LABEL: atomic_sub_i8_soffset__amdgpu_no_remote_memory:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_add_u32 s6, s0, 0x2328
; SI-NEXT: s_addc_u32 s1, s1, 0
; SI-NEXT: s_and_b32 s0, s6, -4
@@ -9641,8 +9493,7 @@ define amdgpu_kernel void @atomic_sub_i8_soffset__amdgpu_no_remote_memory(ptr ad
;
; VI-LABEL: atomic_sub_i8_soffset__amdgpu_no_remote_memory:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_add_u32 s3, s0, 0x2328
@@ -9651,18 +9502,18 @@ define amdgpu_kernel void @atomic_sub_i8_soffset__amdgpu_no_remote_memory(ptr ad
; VI-NEXT: s_load_dword s9, s[0:1], 0x0
; VI-NEXT: s_and_b32 s3, s3, 3
; VI-NEXT: s_lshl_b32 s3, s3, 3
-; VI-NEXT: s_lshl_b32 s6, 0xff, s3
; VI-NEXT: s_and_b32 s2, s2, 0xff
-; VI-NEXT: s_not_b32 s7, s6
-; VI-NEXT: s_lshl_b32 s8, s2, s3
+; VI-NEXT: s_lshl_b32 s6, 0xff, s3
+; VI-NEXT: s_lshl_b32 s7, s2, s3
+; VI-NEXT: s_not_b32 s8, s6
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s9
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: .LBB137_1: ; %atomicrmw.start
; VI-NEXT: ; =>This Inner Loop Header: Depth=1
-; VI-NEXT: v_subrev_u32_e32 v0, vcc, s8, v1
-; VI-NEXT: v_and_b32_e32 v2, s7, v1
+; VI-NEXT: v_subrev_u32_e32 v0, vcc, s7, v1
+; VI-NEXT: v_and_b32_e32 v2, s8, v1
; VI-NEXT: v_and_b32_e32 v0, s6, v0
; VI-NEXT: v_or_b32_e32 v0, v2, v0
; VI-NEXT: v_mov_b32_e32 v3, v1
@@ -9680,44 +9531,41 @@ define amdgpu_kernel void @atomic_sub_i8_soffset__amdgpu_no_remote_memory(ptr ad
;
; GFX9-LABEL: atomic_sub_i8_soffset__amdgpu_no_remote_memory:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[0:1], 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_add_u32 s3, s0, 0x2328
-; GFX9-NEXT: s_addc_u32 s1, s1, 0
-; GFX9-NEXT: s_and_b32 s0, s3, -4
-; GFX9-NEXT: s_load_dword s7, s[0:1], 0x0
-; GFX9-NEXT: s_and_b32 s3, s3, 3
-; GFX9-NEXT: s_lshl_b32 s3, s3, 3
-; GFX9-NEXT: s_lshl_b32 s4, 0xff, s3
-; GFX9-NEXT: s_and_b32 s2, s2, 0xff
-; GFX9-NEXT: s_not_b32 s5, s4
-; GFX9-NEXT: s_lshl_b32 s6, s2, s3
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_add_u32 s4, s8, 0x2328
+; GFX9-NEXT: s_addc_u32 s3, s9, 0
+; GFX9-NEXT: s_and_b32 s2, s4, -4
+; GFX9-NEXT: s_load_dword s7, s[2:3], 0x0
+; GFX9-NEXT: s_and_b32 s4, s4, 3
+; GFX9-NEXT: s_lshl_b32 s6, s4, 3
+; GFX9-NEXT: s_and_b32 s5, s10, 0xff
+; GFX9-NEXT: s_lshl_b32 s4, 0xff, s6
+; GFX9-NEXT: s_lshl_b32 s5, s5, s6
+; GFX9-NEXT: s_not_b32 s6, s4
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s7
; GFX9-NEXT: .LBB137_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_subrev_u32_e32 v0, s6, v1
+; GFX9-NEXT: v_subrev_u32_e32 v0, s5, v1
; GFX9-NEXT: v_and_b32_e32 v0, s4, v0
-; GFX9-NEXT: v_and_or_b32 v0, v1, s5, v0
-; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
+; GFX9-NEXT: v_and_or_b32 v0, v1, s6, v0
+; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[2:3] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX9-NEXT: s_cbranch_execnz .LBB137_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: atomic_sub_i8_soffset__amdgpu_no_remote_memory:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_add_u32 s3, s0, 0x2328
diff --git a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
index fb5e669d680f5..ec893d937c6ea 100644
--- a/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
+++ b/llvm/test/CodeGen/AMDGPU/global_atomics_i32_system.ll
@@ -9118,41 +9118,41 @@ entry:
define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: atomic_min_i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_load_dword s2, s[0:1], 0x0
-; SI-NEXT: s_mov_b64 s[4:5], 0
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dword s3, s[0:1], 0x0
+; SI-NEXT: s_mov_b64 s[8:9], 0
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v1, s2
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v1, s3
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: .LBB130_1: ; %atomicrmw.start
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: v_min_i32_e32 v0, s6, v1
+; SI-NEXT: v_min_i32_e32 v0, s2, v1
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v3, v1
; SI-NEXT: v_mov_b32_e32 v2, v0
-; SI-NEXT: buffer_atomic_cmpswap v[2:3], off, s[0:3], 0 glc
+; SI-NEXT: buffer_atomic_cmpswap v[2:3], off, s[4:7], 0 glc
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: buffer_wbinvl1
; SI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v1
-; SI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
+; SI-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
; SI-NEXT: v_mov_b32_e32 v1, v2
-; SI-NEXT: s_andn2_b64 exec, exec, s[4:5]
+; SI-NEXT: s_andn2_b64 exec, exec, s[8:9]
; SI-NEXT: s_cbranch_execnz .LBB130_1
; SI-NEXT: ; %bb.2: ; %atomicrmw.end
; SI-NEXT: s_endpgm
;
; VI-LABEL: atomic_min_i32:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_load_dword s3, s[6:7], 0x0
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: v_mov_b32_e32 v1, s7
+; VI-NEXT: s_load_dword s3, s[0:1], 0x0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: .LBB130_1: ; %atomicrmw.start
@@ -9162,33 +9162,32 @@ define amdgpu_kernel void @atomic_min_i32(ptr addrspace(1) %out, i32 %in) {
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: buffer_wbinvl1_vol
; VI-NEXT: v_cmp_eq_u32_e32 vcc, v2, v3
-; VI-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
+; VI-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; VI-NEXT: v_mov_b32_e32 v3, v2
-; VI-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; VI-NEXT: s_andn2_b64 exec, exec, s[4:5]
; VI-NEXT: s_cbranch_execnz .LBB130_1
; VI-NEXT: ; %bb.2: ; %atomicrmw.end
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_min_i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b64 s[2:3], 0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dword s4, s[0:1], 0x0
+; GFX9-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: .LBB130_1: ; %atomicrmw.start
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: v_min_i32_e32 v0, s6, v1
+; GFX9-NEXT: v_min_i32_e32 v0, s2, v1
; GFX9-NEXT: global_atomic_cmpswap v0, v2, v[0:1], s[0:1] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: buffer_wbinvl1_vol
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, v0, v1
-; GFX9-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX9-NEXT: s_andn2_b64 exec, exec, s[4:5]
; GFX9-NEXT: s_cbranch_execnz .LBB130_1
; GFX9-NEXT: ; %bb.2: ; %atomicrmw.end
; GFX9-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/half.ll b/llvm/test/CodeGen/AMDGPU/half.ll
index a61d6a6cc071f..62368803457a2 100644
--- a/llvm/test/CodeGen/AMDGPU/half.ll
+++ b/llvm/test/CodeGen/AMDGPU/half.ll
@@ -7,39 +7,22 @@
; half args should be promoted to float for CI and lower.
define amdgpu_kernel void @load_f16_arg(ptr addrspace(1) %out, half %arg) #0 {
-; CI-LABEL: load_f16_arg:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_short v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: load_f16_arg:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: flat_store_short v[0:1], v2
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: load_f16_arg:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s2
+; CIVI-NEXT: flat_store_short v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX11-LABEL: load_f16_arg:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_store_b16 v0, v1, s[0:1]
@@ -49,39 +32,22 @@ define amdgpu_kernel void @load_f16_arg(ptr addrspace(1) %out, half %arg) #0 {
}
define amdgpu_kernel void @load_v2f16_arg(ptr addrspace(1) %out, <2 x half> %arg) #0 {
-; CI-LABEL: load_v2f16_arg:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
-; CI-NEXT: flat_store_dword v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: load_v2f16_arg:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: load_v2f16_arg:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s2
+; CIVI-NEXT: flat_store_dword v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX11-LABEL: load_v2f16_arg:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -98,16 +64,16 @@ define amdgpu_kernel void @load_v3f16_arg(ptr addrspace(1) %out, <3 x half> %arg
; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CIVI-NEXT: s_waitcnt lgkmcnt(0)
-; CIVI-NEXT: s_add_u32 s4, s0, 4
-; CIVI-NEXT: s_addc_u32 s5, s1, 0
-; CIVI-NEXT: v_mov_b32_e32 v2, s4
-; CIVI-NEXT: v_mov_b32_e32 v4, s3
; CIVI-NEXT: v_mov_b32_e32 v0, s0
-; CIVI-NEXT: v_mov_b32_e32 v3, s5
; CIVI-NEXT: v_mov_b32_e32 v1, s1
-; CIVI-NEXT: v_mov_b32_e32 v5, s2
-; CIVI-NEXT: flat_store_short v[2:3], v4
-; CIVI-NEXT: flat_store_dword v[0:1], v5
+; CIVI-NEXT: v_mov_b32_e32 v2, s2
+; CIVI-NEXT: s_add_u32 s0, s0, 4
+; CIVI-NEXT: flat_store_dword v[0:1], v2
+; CIVI-NEXT: s_addc_u32 s1, s1, 0
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s3
+; CIVI-NEXT: flat_store_short v[0:1], v2
; CIVI-NEXT: s_endpgm
;
; GFX11-LABEL: load_v3f16_arg:
@@ -135,8 +101,8 @@ define amdgpu_kernel void @load_v4f16_arg(ptr addrspace(1) %out, <4 x half> %arg
; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CIVI-NEXT: s_waitcnt lgkmcnt(0)
; CIVI-NEXT: v_mov_b32_e32 v0, s0
-; CIVI-NEXT: v_mov_b32_e32 v2, s2
; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: v_mov_b32_e32 v2, s2
; CIVI-NEXT: v_mov_b32_e32 v3, s3
; CIVI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CIVI-NEXT: s_endpgm
@@ -144,9 +110,9 @@ define amdgpu_kernel void @load_v4f16_arg(ptr addrspace(1) %out, <4 x half> %arg
; GFX11-LABEL: load_v4f16_arg:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
store <4 x half> %arg, ptr addrspace(1) %out
@@ -204,43 +170,24 @@ define amdgpu_kernel void @load_v8f16_arg(ptr addrspace(1) %out, <8 x half> %arg
}
define amdgpu_kernel void @extload_v2f16_arg(ptr addrspace(1) %out, <2 x half> %in) #0 {
-; CI-LABEL: extload_v2f16_arg:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s3, s2, 16
-; CI-NEXT: v_cvt_f32_f16_e32 v1, s3
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s2
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: extload_v2f16_arg:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s3, s2, 16
-; VI-NEXT: v_cvt_f32_f16_e32 v1, s3
-; VI-NEXT: v_cvt_f32_f16_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: extload_v2f16_arg:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: s_lshr_b32 s3, s2, 16
+; CIVI-NEXT: v_cvt_f32_f16_e32 v1, s3
+; CIVI-NEXT: v_cvt_f32_f16_e32 v0, s2
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: v_mov_b32_e32 v3, s1
+; CIVI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; CIVI-NEXT: s_endpgm
;
; GFX11-LABEL: extload_v2f16_arg:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_lshr_b32 s3, s2, 16
@@ -254,39 +201,22 @@ define amdgpu_kernel void @extload_v2f16_arg(ptr addrspace(1) %out, <2 x half> %
}
define amdgpu_kernel void @extload_f16_to_f32_arg(ptr addrspace(1) %out, half %arg) #0 {
-; CI-LABEL: extload_f16_to_f32_arg:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_cvt_f32_f16_e32 v2, s2
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: flat_store_dword v[0:1], v2
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: extload_f16_to_f32_arg:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v2, s2
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: extload_f16_to_f32_arg:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_cvt_f32_f16_e32 v2, s2
+; CIVI-NEXT: v_mov_b32_e32 v0, s0
+; CIVI-NEXT: v_mov_b32_e32 v1, s1
+; CIVI-NEXT: flat_store_dword v[0:1], v2
+; CIVI-NEXT: s_endpgm
;
; GFX11-LABEL: extload_f16_to_f32_arg:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cvt_f32_f16_e32 v1, s2
@@ -298,43 +228,24 @@ define amdgpu_kernel void @extload_f16_to_f32_arg(ptr addrspace(1) %out, half %a
}
define amdgpu_kernel void @extload_v2f16_to_v2f32_arg(ptr addrspace(1) %out, <2 x half> %arg) #0 {
-; CI-LABEL: extload_v2f16_to_v2f32_arg:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s3, s2, 16
-; CI-NEXT: v_cvt_f32_f16_e32 v1, s3
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s2
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: extload_v2f16_to_v2f32_arg:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s3, s2, 16
-; VI-NEXT: v_cvt_f32_f16_e32 v1, s3
-; VI-NEXT: v_cvt_f32_f16_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: extload_v2f16_to_v2f32_arg:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: s_lshr_b32 s3, s2, 16
+; CIVI-NEXT: v_cvt_f32_f16_e32 v1, s3
+; CIVI-NEXT: v_cvt_f32_f16_e32 v0, s2
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: v_mov_b32_e32 v3, s1
+; CIVI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; CIVI-NEXT: s_endpgm
;
; GFX11-LABEL: extload_v2f16_to_v2f32_arg:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_lshr_b32 s3, s2, 16
@@ -348,37 +259,21 @@ define amdgpu_kernel void @extload_v2f16_to_v2f32_arg(ptr addrspace(1) %out, <2
}
define amdgpu_kernel void @extload_v3f16_to_v3f32_arg(ptr addrspace(1) %out, <3 x half> %arg) #0 {
-; CI-LABEL: extload_v3f16_to_v3f32_arg:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s4, s2, 16
-; CI-NEXT: v_cvt_f32_f16_e32 v2, s3
-; CI-NEXT: v_cvt_f32_f16_e32 v1, s4
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s2
-; CI-NEXT: v_mov_b32_e32 v4, s1
-; CI-NEXT: v_mov_b32_e32 v3, s0
-; CI-NEXT: flat_store_dwordx3 v[3:4], v[0:2]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: extload_v3f16_to_v3f32_arg:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s4, s2, 16
-; VI-NEXT: v_cvt_f32_f16_e32 v0, s2
-; VI-NEXT: v_cvt_f32_f16_e32 v1, s4
-; VI-NEXT: v_cvt_f32_f16_e32 v2, s3
-; VI-NEXT: v_mov_b32_e32 v4, s1
-; VI-NEXT: v_mov_b32_e32 v3, s0
-; VI-NEXT: flat_store_dwordx3 v[3:4], v[0:2]
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: extload_v3f16_to_v3f32_arg:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: s_lshr_b32 s4, s2, 16
+; CIVI-NEXT: v_cvt_f32_f16_e32 v2, s3
+; CIVI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; CIVI-NEXT: v_cvt_f32_f16_e32 v0, s2
+; CIVI-NEXT: v_mov_b32_e32 v3, s0
+; CIVI-NEXT: v_mov_b32_e32 v4, s1
+; CIVI-NEXT: flat_store_dwordx3 v[3:4], v[0:2]
+; CIVI-NEXT: s_endpgm
;
; GFX11-LABEL: extload_v3f16_to_v3f32_arg:
; GFX11: ; %bb.0:
@@ -386,9 +281,9 @@ define amdgpu_kernel void @extload_v3f16_to_v3f32_arg(ptr addrspace(1) %out, <3
; GFX11-NEXT: v_mov_b32_e32 v3, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_lshr_b32 s4, s2, 16
-; GFX11-NEXT: v_cvt_f32_f16_e32 v0, s2
-; GFX11-NEXT: v_cvt_f32_f16_e32 v1, s4
; GFX11-NEXT: v_cvt_f32_f16_e32 v2, s3
+; GFX11-NEXT: v_cvt_f32_f16_e32 v1, s4
+; GFX11-NEXT: v_cvt_f32_f16_e32 v0, s2
; GFX11-NEXT: global_store_b96 v3, v[0:2], s[0:1]
; GFX11-NEXT: s_endpgm
%ext = fpext <3 x half> %arg to <3 x float>
@@ -410,8 +305,8 @@ define amdgpu_kernel void @extload_v4f16_to_v4f32_arg(ptr addrspace(1) %out, <4
; CI-NEXT: v_cvt_f32_f16_e32 v3, s5
; CI-NEXT: v_cvt_f32_f16_e32 v1, s4
; CI-NEXT: v_cvt_f32_f16_e32 v0, s2
-; CI-NEXT: v_mov_b32_e32 v5, s1
; CI-NEXT: v_mov_b32_e32 v4, s0
+; CI-NEXT: v_mov_b32_e32 v5, s1
; CI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; CI-NEXT: s_endpgm
;
@@ -424,12 +319,12 @@ define amdgpu_kernel void @extload_v4f16_to_v4f32_arg(ptr addrspace(1) %out, <4
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_lshr_b32 s4, s3, 16
; VI-NEXT: s_lshr_b32 s5, s2, 16
-; VI-NEXT: v_cvt_f32_f16_e32 v0, s2
+; VI-NEXT: v_cvt_f32_f16_e32 v2, s3
; VI-NEXT: v_cvt_f32_f16_e32 v3, s4
; VI-NEXT: v_cvt_f32_f16_e32 v1, s5
-; VI-NEXT: v_cvt_f32_f16_e32 v2, s3
-; VI-NEXT: v_mov_b32_e32 v5, s1
+; VI-NEXT: v_cvt_f32_f16_e32 v0, s2
; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: v_mov_b32_e32 v5, s1
; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-NEXT: s_endpgm
;
@@ -440,10 +335,10 @@ define amdgpu_kernel void @extload_v4f16_to_v4f32_arg(ptr addrspace(1) %out, <4
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_lshr_b32 s4, s3, 16
; GFX11-NEXT: s_lshr_b32 s5, s2, 16
-; GFX11-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX11-NEXT: v_cvt_f32_f16_e32 v2, s3
; GFX11-NEXT: v_cvt_f32_f16_e32 v3, s4
; GFX11-NEXT: v_cvt_f32_f16_e32 v1, s5
-; GFX11-NEXT: v_cvt_f32_f16_e32 v2, s3
+; GFX11-NEXT: v_cvt_f32_f16_e32 v0, s2
; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX11-NEXT: s_endpgm
%ext = fpext <4 x half> %arg to <4 x float>
@@ -543,48 +438,28 @@ define amdgpu_kernel void @extload_v8f16_to_v8f32_arg(ptr addrspace(1) %out, <8
}
define amdgpu_kernel void @extload_f16_to_f64_arg(ptr addrspace(1) %out, half %arg) #0 {
-; CI-LABEL: extload_f16_to_f64_arg:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[8:9], 0x2
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: extload_f16_to_f64_arg:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s0, s[8:9], 0x8
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: extload_f16_to_f64_arg:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: v_cvt_f32_f16_e32 v0, s2
+; CIVI-NEXT: v_mov_b32_e32 v2, s0
+; CIVI-NEXT: v_mov_b32_e32 v3, s1
+; CIVI-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
+; CIVI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; CIVI-NEXT: s_endpgm
;
; GFX11-LABEL: extload_f16_to_f64_arg:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_f16_e32 v0, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: v_cvt_f32_f16_e32 v0, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f64_f32_e32 v[0:1], v0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
%ext = fpext half %arg to double
@@ -593,57 +468,34 @@ define amdgpu_kernel void @extload_f16_to_f64_arg(ptr addrspace(1) %out, half %a
}
define amdgpu_kernel void @extload_v2f16_to_v2f64_arg(ptr addrspace(1) %out, <2 x half> %arg) #0 {
-; CI-LABEL: extload_v2f16_to_v2f64_arg:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[8:9], 0x2
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s1, s0, 16
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s1
-; CI-NEXT: v_cvt_f32_f16_e32 v1, s0
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: v_cvt_f64_f32_e32 v[2:3], v0
-; CI-NEXT: v_cvt_f64_f32_e32 v[0:1], v1
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v5, s1
-; CI-NEXT: v_mov_b32_e32 v4, s0
-; CI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: extload_v2f16_to_v2f64_arg:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s0, s[8:9], 0x8
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s1, s0, 16
-; VI-NEXT: v_cvt_f32_f16_e32 v0, s1
-; VI-NEXT: v_cvt_f32_f16_e32 v1, s0
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: v_cvt_f64_f32_e32 v[2:3], v0
-; VI-NEXT: v_cvt_f64_f32_e32 v[0:1], v1
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v5, s1
-; VI-NEXT: v_mov_b32_e32 v4, s0
-; VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; VI-NEXT: s_endpgm
+; CIVI-LABEL: extload_v2f16_to_v2f64_arg:
+; CIVI: ; %bb.0:
+; CIVI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CIVI-NEXT: s_add_i32 s12, s12, s17
+; CIVI-NEXT: s_mov_b32 flat_scratch_lo, s13
+; CIVI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; CIVI-NEXT: s_waitcnt lgkmcnt(0)
+; CIVI-NEXT: s_lshr_b32 s3, s2, 16
+; CIVI-NEXT: v_cvt_f32_f16_e32 v0, s3
+; CIVI-NEXT: v_cvt_f32_f16_e32 v1, s2
+; CIVI-NEXT: v_mov_b32_e32 v4, s0
+; CIVI-NEXT: v_mov_b32_e32 v5, s1
+; CIVI-NEXT: v_cvt_f64_f32_e32 v[2:3], v0
+; CIVI-NEXT: v_cvt_f64_f32_e32 v[0:1], v1
+; CIVI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; CIVI-NEXT: s_endpgm
;
; GFX11-LABEL: extload_v2f16_to_v2f64_arg:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v4, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s1, s0, 16
-; GFX11-NEXT: v_cvt_f32_f16_e32 v1, s0
-; GFX11-NEXT: v_cvt_f32_f16_e32 v0, s1
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-NEXT: v_cvt_f32_f16_e32 v1, s2
+; GFX11-NEXT: v_cvt_f32_f16_e32 v0, s3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_cvt_f64_f32_e32 v[2:3], v0
; GFX11-NEXT: v_cvt_f64_f32_e32 v[0:1], v1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX11-NEXT: s_endpgm
%ext = fpext <2 x half> %arg to <2 x double>
@@ -3116,18 +2968,16 @@ define amdgpu_kernel void @global_truncstore_v16f32_to_v16f16(ptr addrspace(1) %
define amdgpu_kernel void @fadd_f16(ptr addrspace(1) %out, half %a, half %b) #0 {
; CI-LABEL: fadd_f16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s0, s[8:9], 0x2
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s1, s0, 16
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s1
-; CI-NEXT: v_cvt_f32_f16_e32 v1, s0
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_lshr_b32 s3, s2, 16
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s3
+; CI-NEXT: v_cvt_f32_f16_e32 v1, s2
; CI-NEXT: v_add_f32_e32 v0, v1, v0
; CI-NEXT: v_cvt_f16_f32_e32 v2, v0
-; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
; CI-NEXT: flat_store_short v[0:1], v2
@@ -3135,25 +2985,22 @@ define amdgpu_kernel void @fadd_f16(ptr addrspace(1) %out, half %a, half %b) #0
;
; VI-LABEL: fadd_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s3, s2, 16
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_add_f16_e32 v2, s2, v0
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_lshr_b32 s0, s2, 16
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_add_f16_e32 v2, s2, v2
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: fadd_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -3164,9 +3011,7 @@ define amdgpu_kernel void @fadd_f16(ptr addrspace(1) %out, half %a, half %b) #0
;
; GFX11-FAKE16-LABEL: fadd_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
@@ -3189,18 +3034,18 @@ define amdgpu_kernel void @fadd_v2f16(ptr addrspace(1) %out, <2 x half> %a, <2 x
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: s_lshr_b32 s4, s2, 16
; CI-NEXT: s_lshr_b32 s5, s3, 16
-; CI-NEXT: v_cvt_f32_f16_e32 v0, s3
-; CI-NEXT: v_cvt_f32_f16_e32 v1, s2
-; CI-NEXT: v_cvt_f32_f16_e32 v2, s5
-; CI-NEXT: v_cvt_f32_f16_e32 v3, s4
+; CI-NEXT: v_cvt_f32_f16_e32 v0, s5
+; CI-NEXT: v_cvt_f32_f16_e32 v1, s4
+; CI-NEXT: v_cvt_f32_f16_e32 v2, s3
+; CI-NEXT: v_cvt_f32_f16_e32 v3, s2
; CI-NEXT: v_add_f32_e32 v0, v1, v0
-; CI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; CI-NEXT: v_add_f32_e32 v1, v3, v2
-; CI-NEXT: v_cvt_f16_f32_e32 v1, v1
-; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_or_b32_e32 v2, v0, v1
+; CI-NEXT: v_cvt_f16_f32_e32 v4, v0
+; CI-NEXT: v_add_f32_e32 v0, v3, v2
+; CI-NEXT: v_cvt_f16_f32_e32 v2, v0
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; CI-NEXT: v_mov_b32_e32 v1, s1
+; CI-NEXT: v_or_b32_e32 v2, v2, v3
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
@@ -3211,16 +3056,16 @@ define amdgpu_kernel void @fadd_v2f16(ptr addrspace(1) %out, <2 x half> %a, <2 x
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s4, s3, 16
-; VI-NEXT: s_lshr_b32 s5, s2, 16
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_mov_b32_e32 v1, s4
-; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: v_add_f16_e32 v0, s2, v0
-; VI-NEXT: v_add_f16_sdwa v1, v2, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-NEXT: v_or_b32_e32 v2, v0, v1
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_lshr_b32 s0, s3, 16
+; VI-NEXT: s_lshr_b32 s1, s2, 16
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_add_f16_sdwa v2, v3, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: v_add_f16_e32 v3, s2, v3
+; VI-NEXT: v_or_b32_e32 v2, v3, v2
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
index 1f751ecbe6645..95ea38f7fb43a 100644
--- a/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
+++ b/llvm/test/CodeGen/AMDGPU/idiv-licm.ll
@@ -6,133 +6,129 @@
define amdgpu_kernel void @udiv32_invariant_denom(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GFX9-LABEL: udiv32_invariant_denom:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s5, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX9-NEXT: s_sub_i32 s2, 0, s6
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT: s_sub_i32 s3, 0, s2
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: s_mul_i32 s2, s2, s4
-; GFX9-NEXT: s_mul_hi_u32 s2, s4, s2
-; GFX9-NEXT: s_add_i32 s4, s4, s2
-; GFX9-NEXT: s_mov_b32 s2, s3
+; GFX9-NEXT: s_mul_i32 s3, s3, s4
+; GFX9-NEXT: s_mul_hi_u32 s3, s4, s3
+; GFX9-NEXT: s_add_i32 s3, s4, s3
+; GFX9-NEXT: s_mov_b32 s4, s5
; GFX9-NEXT: .LBB0_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_mul_hi_u32 s5, s2, s4
-; GFX9-NEXT: s_mul_i32 s7, s5, s6
-; GFX9-NEXT: s_sub_i32 s7, s2, s7
-; GFX9-NEXT: s_add_i32 s8, s5, 1
-; GFX9-NEXT: s_sub_i32 s9, s7, s6
-; GFX9-NEXT: s_cmp_ge_u32 s7, s6
-; GFX9-NEXT: s_cselect_b32 s5, s8, s5
+; GFX9-NEXT: s_mul_hi_u32 s6, s4, s3
+; GFX9-NEXT: s_mul_i32 s7, s6, s2
+; GFX9-NEXT: s_sub_i32 s7, s4, s7
+; GFX9-NEXT: s_add_i32 s8, s6, 1
+; GFX9-NEXT: s_sub_i32 s9, s7, s2
+; GFX9-NEXT: s_cmp_ge_u32 s7, s2
+; GFX9-NEXT: s_cselect_b32 s6, s8, s6
; GFX9-NEXT: s_cselect_b32 s7, s9, s7
-; GFX9-NEXT: s_add_i32 s8, s5, 1
-; GFX9-NEXT: s_cmp_ge_u32 s7, s6
-; GFX9-NEXT: s_cselect_b32 s5, s8, s5
-; GFX9-NEXT: s_lshl_b64 s[8:9], s[2:3], 2
-; GFX9-NEXT: s_add_u32 s8, s0, s8
-; GFX9-NEXT: s_addc_u32 s9, s1, s9
-; GFX9-NEXT: s_add_i32 s2, s2, 1
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: s_cmpk_eq_i32 s2, 0x400
-; GFX9-NEXT: global_store_dword v0, v1, s[8:9]
+; GFX9-NEXT: s_add_i32 s8, s6, 1
+; GFX9-NEXT: s_cmp_ge_u32 s7, s2
+; GFX9-NEXT: s_cselect_b32 s8, s8, s6
+; GFX9-NEXT: s_lshl_b64 s[6:7], s[4:5], 2
+; GFX9-NEXT: s_add_u32 s6, s0, s6
+; GFX9-NEXT: s_addc_u32 s7, s1, s7
+; GFX9-NEXT: s_add_i32 s4, s4, 1
+; GFX9-NEXT: v_mov_b32_e32 v1, s8
+; GFX9-NEXT: s_cmpk_eq_i32 s4, 0x400
+; GFX9-NEXT: global_store_dword v0, v1, s[6:7]
; GFX9-NEXT: s_cbranch_scc0 .LBB0_1
; GFX9-NEXT: ; %bb.2: ; %bb2
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: udiv32_invariant_denom:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX10-NEXT: s_sub_i32 s3, 0, s6
+; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX10-NEXT: s_sub_i32 s4, 0, s2
; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-NEXT: v_readfirstlane_b32 s3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_mul_i32 s3, s3, s2
-; GFX10-NEXT: s_mul_hi_u32 s4, s2, s3
-; GFX10-NEXT: s_mov_b32 s3, 0
-; GFX10-NEXT: s_add_i32 s4, s2, s4
-; GFX10-NEXT: s_mov_b32 s2, s3
+; GFX10-NEXT: s_mul_i32 s4, s4, s3
+; GFX10-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX10-NEXT: s_add_i32 s3, s3, s4
+; GFX10-NEXT: s_mov_b32 s4, s5
; GFX10-NEXT: .LBB0_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_mul_hi_u32 s5, s2, s4
-; GFX10-NEXT: s_mul_i32 s7, s5, s6
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_add_i32 s8, s5, 1
-; GFX10-NEXT: s_sub_i32 s7, s2, s7
-; GFX10-NEXT: s_sub_i32 s9, s7, s6
-; GFX10-NEXT: s_cmp_ge_u32 s7, s6
-; GFX10-NEXT: s_cselect_b32 s5, s8, s5
+; GFX10-NEXT: s_mul_hi_u32 s6, s4, s3
+; GFX10-NEXT: s_mul_i32 s7, s6, s2
+; GFX10-NEXT: s_add_i32 s8, s6, 1
+; GFX10-NEXT: s_sub_i32 s7, s4, s7
+; GFX10-NEXT: s_sub_i32 s9, s7, s2
+; GFX10-NEXT: s_cmp_ge_u32 s7, s2
+; GFX10-NEXT: s_cselect_b32 s6, s8, s6
; GFX10-NEXT: s_cselect_b32 s7, s9, s7
-; GFX10-NEXT: s_add_i32 s8, s5, 1
-; GFX10-NEXT: s_cmp_ge_u32 s7, s6
-; GFX10-NEXT: s_cselect_b32 s5, s8, s5
-; GFX10-NEXT: s_lshl_b64 s[8:9], s[2:3], 2
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: s_add_u32 s8, s0, s8
-; GFX10-NEXT: s_addc_u32 s9, s1, s9
-; GFX10-NEXT: s_add_i32 s2, s2, 1
-; GFX10-NEXT: s_cmpk_eq_i32 s2, 0x400
-; GFX10-NEXT: global_store_dword v0, v1, s[8:9]
+; GFX10-NEXT: s_add_i32 s8, s6, 1
+; GFX10-NEXT: s_cmp_ge_u32 s7, s2
+; GFX10-NEXT: s_cselect_b32 s8, s8, s6
+; GFX10-NEXT: s_lshl_b64 s[6:7], s[4:5], 2
+; GFX10-NEXT: v_mov_b32_e32 v1, s8
+; GFX10-NEXT: s_add_u32 s6, s0, s6
+; GFX10-NEXT: s_addc_u32 s7, s1, s7
+; GFX10-NEXT: s_add_i32 s4, s4, 1
+; GFX10-NEXT: s_cmpk_eq_i32 s4, 0x400
+; GFX10-NEXT: global_store_dword v0, v1, s[6:7]
; GFX10-NEXT: s_cbranch_scc0 .LBB0_1
; GFX10-NEXT: ; %bb.2: ; %bb2
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: udiv32_invariant_denom:
; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX11-NEXT: s_sub_i32 s3, 0, s6
+; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX11-NEXT: s_sub_i32 s4, 0, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_mul_i32 s3, s3, s2
-; GFX11-NEXT: s_mul_hi_u32 s4, s2, s3
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_add_i32 s4, s2, s4
-; GFX11-NEXT: s_mov_b32 s2, s3
+; GFX11-NEXT: s_mul_i32 s4, s4, s3
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mov_b32 s4, s5
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB0_1: ; %bb3
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_hi_u32 s5, s2, s4
-; GFX11-NEXT: s_mul_i32 s7, s5, s6
-; GFX11-NEXT: s_add_i32 s8, s5, 1
-; GFX11-NEXT: s_sub_i32 s7, s2, s7
+; GFX11-NEXT: s_mul_hi_u32 s6, s4, s3
+; GFX11-NEXT: s_mul_i32 s7, s6, s2
+; GFX11-NEXT: s_add_i32 s8, s6, 1
+; GFX11-NEXT: s_sub_i32 s7, s4, s7
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_sub_i32 s9, s7, s6
-; GFX11-NEXT: s_cmp_ge_u32 s7, s6
-; GFX11-NEXT: s_cselect_b32 s5, s8, s5
+; GFX11-NEXT: s_sub_i32 s9, s7, s2
+; GFX11-NEXT: s_cmp_ge_u32 s7, s2
+; GFX11-NEXT: s_cselect_b32 s6, s8, s6
; GFX11-NEXT: s_cselect_b32 s7, s9, s7
-; GFX11-NEXT: s_add_i32 s8, s5, 1
-; GFX11-NEXT: s_cmp_ge_u32 s7, s6
-; GFX11-NEXT: s_cselect_b32 s5, s8, s5
-; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s5
-; GFX11-NEXT: s_add_u32 s8, s0, s8
-; GFX11-NEXT: s_addc_u32 s9, s1, s9
-; GFX11-NEXT: s_add_i32 s2, s2, 1
+; GFX11-NEXT: s_add_i32 s8, s6, 1
+; GFX11-NEXT: s_cmp_ge_u32 s7, s2
+; GFX11-NEXT: s_cselect_b32 s8, s8, s6
+; GFX11-NEXT: s_lshl_b64 s[6:7], s[4:5], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s8
+; GFX11-NEXT: s_add_u32 s6, s0, s6
+; GFX11-NEXT: s_addc_u32 s7, s1, s7
+; GFX11-NEXT: s_add_i32 s4, s4, 1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmpk_eq_i32 s2, 0x400
-; GFX11-NEXT: global_store_b32 v0, v1, s[8:9]
+; GFX11-NEXT: s_cmpk_eq_i32 s4, 0x400
+; GFX11-NEXT: global_store_b32 v0, v1, s[6:7]
; GFX11-NEXT: s_cbranch_scc0 .LBB0_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -156,128 +152,124 @@ bb3: ; preds = %bb3, %bb
define amdgpu_kernel void @urem32_invariant_denom(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GFX9-LABEL: urem32_invariant_denom:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s5, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX9-NEXT: s_sub_i32 s2, 0, s6
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT: s_sub_i32 s3, 0, s2
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v1, v0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: v_readfirstlane_b32 s4, v1
-; GFX9-NEXT: s_mul_i32 s2, s2, s4
-; GFX9-NEXT: s_mul_hi_u32 s2, s4, s2
-; GFX9-NEXT: s_add_i32 s4, s4, s2
-; GFX9-NEXT: s_mov_b32 s2, s3
+; GFX9-NEXT: s_mul_i32 s3, s3, s4
+; GFX9-NEXT: s_mul_hi_u32 s3, s4, s3
+; GFX9-NEXT: s_add_i32 s3, s4, s3
+; GFX9-NEXT: s_mov_b32 s4, s5
; GFX9-NEXT: .LBB1_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_mul_hi_u32 s5, s2, s4
-; GFX9-NEXT: s_mul_i32 s5, s5, s6
-; GFX9-NEXT: s_sub_i32 s5, s2, s5
-; GFX9-NEXT: s_sub_i32 s7, s5, s6
-; GFX9-NEXT: s_cmp_ge_u32 s5, s6
-; GFX9-NEXT: s_cselect_b32 s5, s7, s5
-; GFX9-NEXT: s_sub_i32 s7, s5, s6
-; GFX9-NEXT: s_cmp_ge_u32 s5, s6
-; GFX9-NEXT: s_cselect_b32 s5, s7, s5
-; GFX9-NEXT: s_lshl_b64 s[8:9], s[2:3], 2
-; GFX9-NEXT: s_add_u32 s8, s0, s8
-; GFX9-NEXT: s_addc_u32 s9, s1, s9
-; GFX9-NEXT: s_add_i32 s2, s2, 1
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: s_cmpk_eq_i32 s2, 0x400
-; GFX9-NEXT: global_store_dword v0, v1, s[8:9]
+; GFX9-NEXT: s_mul_hi_u32 s6, s4, s3
+; GFX9-NEXT: s_mul_i32 s6, s6, s2
+; GFX9-NEXT: s_sub_i32 s6, s4, s6
+; GFX9-NEXT: s_sub_i32 s7, s6, s2
+; GFX9-NEXT: s_cmp_ge_u32 s6, s2
+; GFX9-NEXT: s_cselect_b32 s6, s7, s6
+; GFX9-NEXT: s_sub_i32 s7, s6, s2
+; GFX9-NEXT: s_cmp_ge_u32 s6, s2
+; GFX9-NEXT: s_cselect_b32 s8, s7, s6
+; GFX9-NEXT: s_lshl_b64 s[6:7], s[4:5], 2
+; GFX9-NEXT: s_add_u32 s6, s0, s6
+; GFX9-NEXT: s_addc_u32 s7, s1, s7
+; GFX9-NEXT: s_add_i32 s4, s4, 1
+; GFX9-NEXT: v_mov_b32_e32 v1, s8
+; GFX9-NEXT: s_cmpk_eq_i32 s4, 0x400
+; GFX9-NEXT: global_store_dword v0, v1, s[6:7]
; GFX9-NEXT: s_cbranch_scc0 .LBB1_1
; GFX9-NEXT: ; %bb.2: ; %bb2
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: urem32_invariant_denom:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s5, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX10-NEXT: s_sub_i32 s3, 0, s6
+; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX10-NEXT: s_sub_i32 s4, 0, s2
; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX10-NEXT: v_readfirstlane_b32 s2, v0
+; GFX10-NEXT: v_readfirstlane_b32 s3, v0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-NEXT: s_mul_i32 s3, s3, s2
-; GFX10-NEXT: s_mul_hi_u32 s4, s2, s3
-; GFX10-NEXT: s_mov_b32 s3, 0
-; GFX10-NEXT: s_add_i32 s4, s2, s4
-; GFX10-NEXT: s_mov_b32 s2, s3
+; GFX10-NEXT: s_mul_i32 s4, s4, s3
+; GFX10-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX10-NEXT: s_add_i32 s3, s3, s4
+; GFX10-NEXT: s_mov_b32 s4, s5
; GFX10-NEXT: .LBB1_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX10-NEXT: s_mul_hi_u32 s5, s2, s4
-; GFX10-NEXT: s_mul_i32 s5, s5, s6
-; GFX10-NEXT: s_sub_i32 s5, s2, s5
-; GFX10-NEXT: s_sub_i32 s7, s5, s6
-; GFX10-NEXT: s_cmp_ge_u32 s5, s6
-; GFX10-NEXT: s_cselect_b32 s5, s7, s5
-; GFX10-NEXT: s_sub_i32 s7, s5, s6
-; GFX10-NEXT: s_cmp_ge_u32 s5, s6
-; GFX10-NEXT: s_cselect_b32 s5, s7, s5
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_lshl_b64 s[8:9], s[2:3], 2
-; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: s_add_u32 s8, s0, s8
-; GFX10-NEXT: s_addc_u32 s9, s1, s9
-; GFX10-NEXT: s_add_i32 s2, s2, 1
-; GFX10-NEXT: s_cmpk_eq_i32 s2, 0x400
-; GFX10-NEXT: global_store_dword v0, v1, s[8:9]
+; GFX10-NEXT: s_mul_hi_u32 s6, s4, s3
+; GFX10-NEXT: s_mul_i32 s6, s6, s2
+; GFX10-NEXT: s_sub_i32 s6, s4, s6
+; GFX10-NEXT: s_sub_i32 s7, s6, s2
+; GFX10-NEXT: s_cmp_ge_u32 s6, s2
+; GFX10-NEXT: s_cselect_b32 s6, s7, s6
+; GFX10-NEXT: s_sub_i32 s7, s6, s2
+; GFX10-NEXT: s_cmp_ge_u32 s6, s2
+; GFX10-NEXT: s_cselect_b32 s8, s7, s6
+; GFX10-NEXT: s_lshl_b64 s[6:7], s[4:5], 2
+; GFX10-NEXT: v_mov_b32_e32 v1, s8
+; GFX10-NEXT: s_add_u32 s6, s0, s6
+; GFX10-NEXT: s_addc_u32 s7, s1, s7
+; GFX10-NEXT: s_add_i32 s4, s4, 1
+; GFX10-NEXT: s_cmpk_eq_i32 s4, 0x400
+; GFX10-NEXT: global_store_dword v0, v1, s[6:7]
; GFX10-NEXT: s_cbranch_scc0 .LBB1_1
; GFX10-NEXT: ; %bb.2: ; %bb2
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: urem32_invariant_denom:
; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s5, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s6
-; GFX11-NEXT: s_sub_i32 s3, 0, s6
+; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX11-NEXT: s_sub_i32 s4, 0, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_readfirstlane_b32 s2, v0
+; GFX11-NEXT: v_readfirstlane_b32 s3, v0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-NEXT: s_mul_i32 s3, s3, s2
-; GFX11-NEXT: s_mul_hi_u32 s4, s2, s3
-; GFX11-NEXT: s_mov_b32 s3, 0
-; GFX11-NEXT: s_add_i32 s4, s2, s4
-; GFX11-NEXT: s_mov_b32 s2, s3
+; GFX11-NEXT: s_mul_i32 s4, s4, s3
+; GFX11-NEXT: s_mul_hi_u32 s4, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: s_add_i32 s3, s3, s4
+; GFX11-NEXT: s_mov_b32 s4, s5
; GFX11-NEXT: .p2align 6
; GFX11-NEXT: .LBB1_1: ; %bb3
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_hi_u32 s5, s2, s4
-; GFX11-NEXT: s_mul_i32 s5, s5, s6
+; GFX11-NEXT: s_mul_hi_u32 s6, s4, s3
+; GFX11-NEXT: s_mul_i32 s6, s6, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_sub_i32 s5, s2, s5
-; GFX11-NEXT: s_sub_i32 s7, s5, s6
-; GFX11-NEXT: s_cmp_ge_u32 s5, s6
-; GFX11-NEXT: s_cselect_b32 s5, s7, s5
+; GFX11-NEXT: s_sub_i32 s6, s4, s6
+; GFX11-NEXT: s_sub_i32 s7, s6, s2
+; GFX11-NEXT: s_cmp_ge_u32 s6, s2
+; GFX11-NEXT: s_cselect_b32 s6, s7, s6
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_sub_i32 s7, s5, s6
-; GFX11-NEXT: s_cmp_ge_u32 s5, s6
-; GFX11-NEXT: s_cselect_b32 s5, s7, s5
-; GFX11-NEXT: s_lshl_b64 s[8:9], s[2:3], 2
-; GFX11-NEXT: v_mov_b32_e32 v1, s5
-; GFX11-NEXT: s_add_u32 s8, s0, s8
-; GFX11-NEXT: s_addc_u32 s9, s1, s9
-; GFX11-NEXT: s_add_i32 s2, s2, 1
+; GFX11-NEXT: s_sub_i32 s7, s6, s2
+; GFX11-NEXT: s_cmp_ge_u32 s6, s2
+; GFX11-NEXT: s_cselect_b32 s8, s7, s6
+; GFX11-NEXT: s_lshl_b64 s[6:7], s[4:5], 2
+; GFX11-NEXT: v_mov_b32_e32 v1, s8
+; GFX11-NEXT: s_add_u32 s6, s0, s6
+; GFX11-NEXT: s_addc_u32 s7, s1, s7
+; GFX11-NEXT: s_add_i32 s4, s4, 1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_cmpk_eq_i32 s2, 0x400
-; GFX11-NEXT: global_store_b32 v0, v1, s[8:9]
+; GFX11-NEXT: s_cmpk_eq_i32 s4, 0x400
+; GFX11-NEXT: global_store_b32 v0, v1, s[6:7]
; GFX11-NEXT: s_cbranch_scc0 .LBB1_1
; GFX11-NEXT: ; %bb.2: ; %bb2
; GFX11-NEXT: s_endpgm
@@ -301,14 +293,13 @@ bb3: ; preds = %bb3, %bb
define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GFX9-LABEL: sdiv32_invariant_denom:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_abs_i32 s2, s6
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX9-NEXT: s_ashr_i32 s4, s6, 31
-; GFX9-NEXT: s_sub_i32 s5, 0, s2
+; GFX9-NEXT: s_abs_i32 s3, s2
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s3
+; GFX9-NEXT: s_sub_i32 s5, 0, s3
+; GFX9-NEXT: s_ashr_i32 s2, s2, 31
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX9-NEXT: v_cvt_u32_f32_e32 v0, v0
@@ -319,39 +310,37 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: .LBB2_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX9-NEXT: s_mul_hi_u32 s6, s3, s5
-; GFX9-NEXT: s_mul_i32 s7, s6, s2
-; GFX9-NEXT: s_sub_i32 s7, s3, s7
+; GFX9-NEXT: s_mul_hi_u32 s6, s4, s5
+; GFX9-NEXT: s_mul_i32 s7, s6, s3
+; GFX9-NEXT: s_sub_i32 s7, s4, s7
; GFX9-NEXT: s_add_i32 s8, s6, 1
-; GFX9-NEXT: s_sub_i32 s9, s7, s2
-; GFX9-NEXT: s_cmp_ge_u32 s7, s2
+; GFX9-NEXT: s_sub_i32 s9, s7, s3
+; GFX9-NEXT: s_cmp_ge_u32 s7, s3
; GFX9-NEXT: s_cselect_b32 s6, s8, s6
; GFX9-NEXT: s_cselect_b32 s7, s9, s7
; GFX9-NEXT: s_add_i32 s8, s6, 1
-; GFX9-NEXT: s_cmp_ge_u32 s7, s2
+; GFX9-NEXT: s_cmp_ge_u32 s7, s3
; GFX9-NEXT: s_cselect_b32 s6, s8, s6
-; GFX9-NEXT: s_xor_b32 s6, s6, s4
-; GFX9-NEXT: s_sub_i32 s6, s6, s4
-; GFX9-NEXT: s_add_i32 s3, s3, 1
+; GFX9-NEXT: s_xor_b32 s6, s6, s2
+; GFX9-NEXT: s_sub_i32 s6, s6, s2
+; GFX9-NEXT: s_add_i32 s4, s4, 1
; GFX9-NEXT: v_mov_b32_e32 v1, s6
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_add_u32 s0, s0, 4
; GFX9-NEXT: s_addc_u32 s1, s1, 0
-; GFX9-NEXT: s_cmpk_eq_i32 s3, 0x400
+; GFX9-NEXT: s_cmpk_eq_i32 s4, 0x400
; GFX9-NEXT: s_cbranch_scc0 .LBB2_1
; GFX9-NEXT: ; %bb.2: ; %bb2
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: sdiv32_invariant_denom:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s3, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_abs_i32 s2, s3
-; GFX10-NEXT: s_ashr_i32 s3, s3, 31
-; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX10-NEXT: s_sub_i32 s4, 0, s2
+; GFX10-NEXT: s_abs_i32 s3, s2
+; GFX10-NEXT: s_ashr_i32 s2, s2, 31
+; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s3
+; GFX10-NEXT: s_sub_i32 s4, 0, s3
; GFX10-NEXT: v_rcp_f32_e32 v0, v0
; GFX10-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX10-NEXT: v_cvt_u32_f32_e32 v0, v0
@@ -364,19 +353,19 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: .LBB2_1: ; %bb3
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_mul_hi_u32 s6, s4, s5
-; GFX10-NEXT: s_mul_i32 s7, s6, s2
+; GFX10-NEXT: s_mul_i32 s7, s6, s3
; GFX10-NEXT: s_add_i32 s8, s6, 1
; GFX10-NEXT: s_sub_i32 s7, s4, s7
-; GFX10-NEXT: s_sub_i32 s9, s7, s2
-; GFX10-NEXT: s_cmp_ge_u32 s7, s2
+; GFX10-NEXT: s_sub_i32 s9, s7, s3
+; GFX10-NEXT: s_cmp_ge_u32 s7, s3
; GFX10-NEXT: s_cselect_b32 s6, s8, s6
; GFX10-NEXT: s_cselect_b32 s7, s9, s7
; GFX10-NEXT: s_add_i32 s8, s6, 1
-; GFX10-NEXT: s_cmp_ge_u32 s7, s2
+; GFX10-NEXT: s_cmp_ge_u32 s7, s3
; GFX10-NEXT: s_cselect_b32 s6, s8, s6
; GFX10-NEXT: s_add_i32 s4, s4, 1
-; GFX10-NEXT: s_xor_b32 s6, s6, s3
-; GFX10-NEXT: s_sub_i32 s6, s6, s3
+; GFX10-NEXT: s_xor_b32 s6, s6, s2
+; GFX10-NEXT: s_sub_i32 s6, s6, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s6
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
@@ -389,14 +378,12 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
;
; GFX11-LABEL: sdiv32_invariant_denom:
; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_abs_i32 s2, s3
-; GFX11-NEXT: s_ashr_i32 s3, s3, 31
-; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX11-NEXT: s_sub_i32 s4, 0, s2
+; GFX11-NEXT: s_abs_i32 s3, s2
+; GFX11-NEXT: s_ashr_i32 s2, s2, 31
+; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s3
+; GFX11-NEXT: s_sub_i32 s4, 0, s3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -414,21 +401,21 @@ define amdgpu_kernel void @sdiv32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: s_mul_hi_u32 s6, s4, s5
-; GFX11-NEXT: s_mul_i32 s7, s6, s2
+; GFX11-NEXT: s_mul_i32 s7, s6, s3
; GFX11-NEXT: s_add_i32 s8, s6, 1
; GFX11-NEXT: s_sub_i32 s7, s4, s7
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: s_sub_i32 s9, s7, s2
-; GFX11-NEXT: s_cmp_ge_u32 s7, s2
+; GFX11-NEXT: s_sub_i32 s9, s7, s3
+; GFX11-NEXT: s_cmp_ge_u32 s7, s3
; GFX11-NEXT: s_cselect_b32 s6, s8, s6
; GFX11-NEXT: s_cselect_b32 s7, s9, s7
; GFX11-NEXT: s_add_i32 s8, s6, 1
-; GFX11-NEXT: s_cmp_ge_u32 s7, s2
+; GFX11-NEXT: s_cmp_ge_u32 s7, s3
; GFX11-NEXT: s_cselect_b32 s6, s8, s6
; GFX11-NEXT: s_add_i32 s4, s4, 1
-; GFX11-NEXT: s_xor_b32 s6, s6, s3
+; GFX11-NEXT: s_xor_b32 s6, s6, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_sub_i32 s6, s6, s3
+; GFX11-NEXT: s_sub_i32 s6, s6, s2
; GFX11-NEXT: v_mov_b32_e32 v1, s6
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_add_u32 s0, s0, 4
@@ -457,12 +444,11 @@ bb3: ; preds = %bb3, %bb
define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GFX9-LABEL: srem32_invariant_denom:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_abs_i32 s2, s0
+; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_abs_i32 s2, s2
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: s_sub_i32 s4, 0, s2
; GFX9-NEXT: v_rcp_f32_e32 v0, v0
; GFX9-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
@@ -485,7 +471,6 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: s_cselect_b32 s5, s6, s5
; GFX9-NEXT: s_add_i32 s3, s3, 1
; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_add_u32 s0, s0, 4
; GFX9-NEXT: s_addc_u32 s1, s1, 0
@@ -496,10 +481,9 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
;
; GFX10-LABEL: srem32_invariant_denom:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_abs_i32 s2, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_abs_i32 s2, s2
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2
; GFX10-NEXT: s_sub_i32 s3, 0, s2
; GFX10-NEXT: v_rcp_f32_e32 v0, v0
@@ -524,7 +508,6 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX10-NEXT: s_cselect_b32 s5, s6, s5
; GFX10-NEXT: s_add_i32 s3, s3, 1
; GFX10-NEXT: v_mov_b32_e32 v1, s5
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_add_u32 s0, s0, 4
@@ -536,21 +519,21 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
;
; GFX11-LABEL: srem32_invariant_denom:
; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_abs_i32 s2, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_abs_i32 s2, s2
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s2
; GFX11-NEXT: s_sub_i32 s3, 0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_rcp_f32_e32 v0, v0
; GFX11-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX11-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
; GFX11-NEXT: v_readfirstlane_b32 s4, v0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_mul_i32 s3, s3, s4
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: s_mul_hi_u32 s5, s4, s3
; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: s_add_i32 s4, s4, s5
@@ -571,7 +554,6 @@ define amdgpu_kernel void @srem32_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX11-NEXT: s_cselect_b32 s5, s6, s5
; GFX11-NEXT: s_add_i32 s3, s3, 1
; GFX11-NEXT: v_mov_b32_e32 v1, s5
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_add_u32 s0, s0, 4
; GFX11-NEXT: s_addc_u32 s1, s1, 0
@@ -599,12 +581,11 @@ bb3: ; preds = %bb3, %bb
define amdgpu_kernel void @udiv16_invariant_denom(ptr addrspace(1) nocapture %arg, i16 %arg1) {
; GFX9-LABEL: udiv16_invariant_denom:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b32 s2, 0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
-; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s0
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
+; GFX9-NEXT: s_mov_b32 s2, 0
; GFX9-NEXT: v_rcp_f32_e32 v1, v0
; GFX9-NEXT: .LBB4_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -621,7 +602,6 @@ define amdgpu_kernel void @udiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: v_mov_b32_e32 v3, s3
; GFX9-NEXT: s_cmpk_eq_i32 s4, 0x400
; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v5, vcc
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v3, v2, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB4_1
; GFX9-NEXT: ; %bb.2: ; %bb2
@@ -629,9 +609,7 @@ define amdgpu_kernel void @udiv16_invariant_denom(ptr addrspace(1) nocapture %ar
;
; GFX10-LABEL: udiv16_invariant_denom:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_and_b32 s2, s2, 0xffff
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2
@@ -659,9 +637,7 @@ define amdgpu_kernel void @udiv16_invariant_denom(ptr addrspace(1) nocapture %ar
;
; GFX11-LABEL: udiv16_invariant_denom:
; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s2, s2, 0xffff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
@@ -712,12 +688,11 @@ bb3: ; preds = %bb3, %bb
define amdgpu_kernel void @urem16_invariant_denom(ptr addrspace(1) nocapture %arg, i16 %arg1) {
; GFX9-LABEL: urem16_invariant_denom:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_and_b32 s2, s0, 0xffff
+; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
; GFX9-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_rcp_f32_e32 v1, v0
; GFX9-NEXT: .LBB5_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -736,7 +711,6 @@ define amdgpu_kernel void @urem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: v_mul_lo_u32 v2, v2, s2
; GFX9-NEXT: s_cmpk_eq_i32 s6, 0x400
; GFX9-NEXT: v_sub_u32_e32 v2, s4, v2
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v3, v2, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB5_1
; GFX9-NEXT: ; %bb.2: ; %bb2
@@ -744,11 +718,9 @@ define amdgpu_kernel void @urem16_invariant_denom(ptr addrspace(1) nocapture %ar
;
; GFX10-LABEL: urem16_invariant_denom:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-NEXT: s_mov_b32 s3, 0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: s_and_b32 s2, s2, 0xffff
; GFX10-NEXT: v_cvt_f32_u32_e32 v0, s2
; GFX10-NEXT: v_rcp_f32_e32 v1, v0
@@ -776,11 +748,9 @@ define amdgpu_kernel void @urem16_invariant_denom(ptr addrspace(1) nocapture %ar
;
; GFX11-LABEL: urem16_invariant_denom:
; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_mov_b32 s3, 0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: s_and_b32 s2, s2, 0xffff
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_u32_e32 v0, s2
@@ -832,12 +802,11 @@ bb3: ; preds = %bb3, %bb
define amdgpu_kernel void @sdiv16_invariant_denom(ptr addrspace(1) nocapture %arg, i16 %arg1) {
; GFX9-LABEL: sdiv16_invariant_denom:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s2, s0
+; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_sext_i32_i16 s2, s2
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_rcp_f32_e32 v1, v0
; GFX9-NEXT: .LBB6_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -860,7 +829,6 @@ define amdgpu_kernel void @sdiv16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: s_and_b32 s5, s3, 0xffff
; GFX9-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NEXT: s_cmpk_eq_i32 s5, 0x400
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v3, v2, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB6_1
; GFX9-NEXT: ; %bb.2: ; %bb2
@@ -868,11 +836,9 @@ define amdgpu_kernel void @sdiv16_invariant_denom(ptr addrspace(1) nocapture %ar
;
; GFX10-LABEL: sdiv16_invariant_denom:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-NEXT: s_mov_b32 s3, 0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: s_sext_i32_i16 s2, s2
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, s2
; GFX10-NEXT: v_rcp_f32_e32 v1, v0
@@ -904,11 +870,9 @@ define amdgpu_kernel void @sdiv16_invariant_denom(ptr addrspace(1) nocapture %ar
;
; GFX11-LABEL: sdiv16_invariant_denom:
; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_mov_b32 s3, 0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: s_sext_i32_i16 s2, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, s2
@@ -963,12 +927,11 @@ bb3: ; preds = %bb3, %bb
define amdgpu_kernel void @srem16_invariant_denom(ptr addrspace(1) nocapture %arg, i16 %arg1) {
; GFX9-LABEL: srem16_invariant_denom:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_sext_i32_i16 s2, s0
+; GFX9-NEXT: s_mov_b32 s3, 0
+; GFX9-NEXT: s_sext_i32_i16 s2, s2
; GFX9-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NEXT: v_rcp_f32_e32 v1, v0
; GFX9-NEXT: .LBB7_1: ; %bb3
; GFX9-NEXT: ; =>This Inner Loop Header: Depth=1
@@ -993,7 +956,6 @@ define amdgpu_kernel void @srem16_invariant_denom(ptr addrspace(1) nocapture %ar
; GFX9-NEXT: v_mov_b32_e32 v3, s4
; GFX9-NEXT: s_cmpk_eq_i32 s5, 0x400
; GFX9-NEXT: v_sub_u32_e32 v2, s6, v2
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_store_short v3, v2, s[0:1]
; GFX9-NEXT: s_cbranch_scc0 .LBB7_1
; GFX9-NEXT: ; %bb.2: ; %bb2
@@ -1001,11 +963,9 @@ define amdgpu_kernel void @srem16_invariant_denom(ptr addrspace(1) nocapture %ar
;
; GFX10-LABEL: srem16_invariant_denom:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-NEXT: s_mov_b32 s3, 0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s3, 0
; GFX10-NEXT: s_sext_i32_i16 s2, s2
; GFX10-NEXT: v_cvt_f32_i32_e32 v0, s2
; GFX10-NEXT: v_rcp_f32_e32 v1, v0
@@ -1039,11 +999,9 @@ define amdgpu_kernel void @srem16_invariant_denom(ptr addrspace(1) nocapture %ar
;
; GFX11-LABEL: srem16_invariant_denom:
; GFX11: ; %bb.0: ; %bb
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_mov_b32 s3, 0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s3, 0
; GFX11-NEXT: s_sext_i32_i16 s2, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_cvt_f32_i32_e32 v0, s2
diff --git a/llvm/test/CodeGen/AMDGPU/iglp-no-clobber.ll b/llvm/test/CodeGen/AMDGPU/iglp-no-clobber.ll
index 9843774d55455..63014f6f94052 100644
--- a/llvm/test/CodeGen/AMDGPU/iglp-no-clobber.ll
+++ b/llvm/test/CodeGen/AMDGPU/iglp-no-clobber.ll
@@ -7,32 +7,31 @@
define amdgpu_kernel void @func(ptr addrspace(1) %in, ptr addrspace(3) %out) {
; CHECK-LABEL: func:
; CHECK: ; %bb.0: ; %.lr.ph
-; CHECK-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; CHECK-NEXT: s_mov_b64 s[8:9], 0
-; CHECK-NEXT: s_mov_b32 s12, 32
-; CHECK-NEXT: s_mov_b32 s2, 0
-; CHECK-NEXT: s_mov_b64 s[0:1], s[8:9]
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s3, 32
+; CHECK-NEXT: s_mov_b32 s6, 0
+; CHECK-NEXT: s_mov_b64 s[4:5], s[8:9]
; CHECK-NEXT: s_mov_b64 s[10:11], 0
; CHECK-NEXT: .LBB0_1: ; %loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: s_and_b32 s1, s1, 0xffff
-; CHECK-NEXT: s_mov_b32 s3, s2
-; CHECK-NEXT: buffer_load_dwordx2 v[0:1], off, s[0:3], 0
+; CHECK-NEXT: s_and_b32 s5, s5, 0xffff
+; CHECK-NEXT: s_mov_b32 s7, s6
+; CHECK-NEXT: buffer_load_dwordx2 v[0:1], off, s[4:7], 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_add_u32 s0, s6, s10
-; CHECK-NEXT: s_addc_u32 s1, s7, s11
+; CHECK-NEXT: s_add_u32 s4, s0, s10
+; CHECK-NEXT: s_addc_u32 s5, s1, s11
; CHECK-NEXT: s_load_dwordx2 s[10:11], s[8:9], 0x0
-; CHECK-NEXT: s_add_i32 s12, s12, -1
-; CHECK-NEXT: s_cmp_lg_u32 s12, 0
+; CHECK-NEXT: s_add_i32 s3, s3, -1
+; CHECK-NEXT: s_cmp_lg_u32 s3, 0
; CHECK-NEXT: ; iglp_opt mask(0x00000000)
; CHECK-NEXT: s_cbranch_scc1 .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %end
-; CHECK-NEXT: s_load_dword s0, s[4:5], 0x8
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: v_and_b32_e32 v0, 0xff, v0
; CHECK-NEXT: v_mov_b32_e32 v1, 0
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v2, s0
+; CHECK-NEXT: v_mov_b32_e32 v2, s2
; CHECK-NEXT: ds_write_b64 v2, v[0:1]
; CHECK-NEXT: s_endpgm
.lr.ph:
diff --git a/llvm/test/CodeGen/AMDGPU/imm.ll b/llvm/test/CodeGen/AMDGPU/imm.ll
index aea3375f0e8bd..9048125decec3 100644
--- a/llvm/test/CodeGen/AMDGPU/imm.ll
+++ b/llvm/test/CodeGen/AMDGPU/imm.ll
@@ -597,35 +597,38 @@ define amdgpu_kernel void @store_literal_imm_f32(ptr addrspace(1) %out) {
define amdgpu_kernel void @add_inline_imm_0.0_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_0.0_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, 0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_0.0_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, 0
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, 0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_0.0_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, 0
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, 0
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, 0.0
store float %y, ptr addrspace(1) %out
@@ -635,35 +638,38 @@ define amdgpu_kernel void @add_inline_imm_0.0_f32(ptr addrspace(1) %out, float %
define amdgpu_kernel void @add_inline_imm_0.5_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_0.5_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, 0.5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, 0.5
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_0.5_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, 0.5
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, 0.5
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_0.5_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, 0.5
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, 0.5
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, 0.5
store float %y, ptr addrspace(1) %out
@@ -673,35 +679,38 @@ define amdgpu_kernel void @add_inline_imm_0.5_f32(ptr addrspace(1) %out, float %
define amdgpu_kernel void @add_inline_imm_neg_0.5_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_neg_0.5_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, -0.5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, -0.5
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_neg_0.5_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, -0.5
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, -0.5
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_neg_0.5_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, -0.5
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, -0.5
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, -0.5
store float %y, ptr addrspace(1) %out
@@ -711,35 +720,38 @@ define amdgpu_kernel void @add_inline_imm_neg_0.5_f32(ptr addrspace(1) %out, flo
define amdgpu_kernel void @add_inline_imm_1.0_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_1.0_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, 1.0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, 1.0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_1.0_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, 1.0
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, 1.0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_1.0_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, 1.0
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, 1.0
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, 1.0
store float %y, ptr addrspace(1) %out
@@ -749,35 +761,38 @@ define amdgpu_kernel void @add_inline_imm_1.0_f32(ptr addrspace(1) %out, float %
define amdgpu_kernel void @add_inline_imm_neg_1.0_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_neg_1.0_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, -1.0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, -1.0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_neg_1.0_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, -1.0
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, -1.0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_neg_1.0_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, -1.0
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, -1.0
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, -1.0
store float %y, ptr addrspace(1) %out
@@ -787,35 +802,38 @@ define amdgpu_kernel void @add_inline_imm_neg_1.0_f32(ptr addrspace(1) %out, flo
define amdgpu_kernel void @add_inline_imm_2.0_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_2.0_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, 2.0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, 2.0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_2.0_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, 2.0
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, 2.0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_2.0_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, 2.0
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, 2.0
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, 2.0
store float %y, ptr addrspace(1) %out
@@ -825,35 +843,38 @@ define amdgpu_kernel void @add_inline_imm_2.0_f32(ptr addrspace(1) %out, float %
define amdgpu_kernel void @add_inline_imm_neg_2.0_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_neg_2.0_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, -2.0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, -2.0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_neg_2.0_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, -2.0
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, -2.0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_neg_2.0_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, -2.0
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, -2.0
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, -2.0
store float %y, ptr addrspace(1) %out
@@ -863,35 +884,38 @@ define amdgpu_kernel void @add_inline_imm_neg_2.0_f32(ptr addrspace(1) %out, flo
define amdgpu_kernel void @add_inline_imm_4.0_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_4.0_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, 4.0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, 4.0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_4.0_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, 4.0
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, 4.0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_4.0_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, 4.0
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, 4.0
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, 4.0
store float %y, ptr addrspace(1) %out
@@ -901,35 +925,38 @@ define amdgpu_kernel void @add_inline_imm_4.0_f32(ptr addrspace(1) %out, float %
define amdgpu_kernel void @add_inline_imm_neg_4.0_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_neg_4.0_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, -4.0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, -4.0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_neg_4.0_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, -4.0
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, -4.0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_neg_4.0_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, -4.0
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, -4.0
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, -4.0
store float %y, ptr addrspace(1) %out
@@ -1059,35 +1086,38 @@ define amdgpu_kernel void @commute_add_literal_f32(ptr addrspace(1) %out, ptr ad
define amdgpu_kernel void @add_inline_imm_1_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_1_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, 1
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, 1
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_1_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, 1
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, 1
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_1_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, 1
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, 1
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, 0x36a0000000000000
store float %y, ptr addrspace(1) %out
@@ -1097,35 +1127,38 @@ define amdgpu_kernel void @add_inline_imm_1_f32(ptr addrspace(1) %out, float %x)
define amdgpu_kernel void @add_inline_imm_2_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_2_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, 2
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, 2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_2_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, 2
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, 2
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_2_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, 2
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, 2
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, 0x36b0000000000000
store float %y, ptr addrspace(1) %out
@@ -1135,35 +1168,38 @@ define amdgpu_kernel void @add_inline_imm_2_f32(ptr addrspace(1) %out, float %x)
define amdgpu_kernel void @add_inline_imm_16_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_16_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, 16
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, 16
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_16_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, 16
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, 16
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_16_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, 16
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, 16
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, 0x36e0000000000000
store float %y, ptr addrspace(1) %out
@@ -1173,38 +1209,41 @@ define amdgpu_kernel void @add_inline_imm_16_f32(ptr addrspace(1) %out, float %x
define amdgpu_kernel void @add_inline_imm_neg_1_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_neg_1_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_add_i32 s4, s6, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_add_i32 s0, s2, -1
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_neg_1_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_i32 s4, s6, -1
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_add_i32 s0, s2, -1
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_neg_1_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_add_i32 s4, s6, -1
-; GFX942-NEXT: v_mov_b32_e32 v0, s4
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_add_i32 s0, s2, -1
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%xbc = bitcast float %x to i32
%y = add i32 %xbc, -1
@@ -1216,38 +1255,41 @@ define amdgpu_kernel void @add_inline_imm_neg_1_f32(ptr addrspace(1) %out, float
define amdgpu_kernel void @add_inline_imm_neg_2_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_neg_2_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_add_i32 s4, s6, -2
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_add_i32 s0, s2, -2
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_neg_2_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_i32 s4, s6, -2
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_add_i32 s0, s2, -2
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_neg_2_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_add_i32 s4, s6, -2
-; GFX942-NEXT: v_mov_b32_e32 v0, s4
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_add_i32 s0, s2, -2
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%xbc = bitcast float %x to i32
%y = add i32 %xbc, -2
@@ -1259,38 +1301,41 @@ define amdgpu_kernel void @add_inline_imm_neg_2_f32(ptr addrspace(1) %out, float
define amdgpu_kernel void @add_inline_imm_neg_16_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_neg_16_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_add_i32 s4, s6, -16
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_add_i32 s0, s2, -16
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_neg_16_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_i32 s4, s6, -16
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_add_i32 s0, s2, -16
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_neg_16_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_add_i32 s4, s6, -16
-; GFX942-NEXT: v_mov_b32_e32 v0, s4
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_add_i32 s0, s2, -16
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_mov_b32_e32 v0, s0
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%xbc = bitcast float %x to i32
%y = add i32 %xbc, -16
@@ -1302,35 +1347,38 @@ define amdgpu_kernel void @add_inline_imm_neg_16_f32(ptr addrspace(1) %out, floa
define amdgpu_kernel void @add_inline_imm_63_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_63_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, 63
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, 63
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_63_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, 63
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, 63
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_63_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, 63
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, 63
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, 0x36ff800000000000
store float %y, ptr addrspace(1) %out
@@ -1340,35 +1388,38 @@ define amdgpu_kernel void @add_inline_imm_63_f32(ptr addrspace(1) %out, float %x
define amdgpu_kernel void @add_inline_imm_64_f32(ptr addrspace(1) %out, float %x) {
; SI-LABEL: add_inline_imm_64_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e64 v0, s6, 64
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_add_f32_e64 v0, s2, 64
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: add_inline_imm_64_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e64 v0, s6, 64
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_add_f32_e64 v0, s2, 64
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX942-LABEL: add_inline_imm_64_f32:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX942-NEXT: s_mov_b32 s3, 0xf000
-; GFX942-NEXT: s_mov_b32 s2, -1
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_mov_b32 s7, 0xf000
+; GFX942-NEXT: s_mov_b32 s6, -1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_add_f32_e64 v0, s6, 64
-; GFX942-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX942-NEXT: s_mov_b32 s4, s0
+; GFX942-NEXT: s_mov_b32 s5, s1
+; GFX942-NEXT: v_add_f32_e64 v0, s2, 64
+; GFX942-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX942-NEXT: s_endpgm
%y = fadd float %x, 0x3700000000000000
store float %y, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/imm16.ll b/llvm/test/CodeGen/AMDGPU/imm16.ll
index 0b2e5d540d463..ebb7bc9961cff 100644
--- a/llvm/test/CodeGen/AMDGPU/imm16.ll
+++ b/llvm/test/CodeGen/AMDGPU/imm16.ll
@@ -771,11 +771,9 @@ define amdgpu_kernel void @store_literal_imm_f16(ptr addrspace(1) %out) {
define amdgpu_kernel void @add_inline_imm_0.0_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_0.0_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, 0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x00,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -783,11 +781,9 @@ define amdgpu_kernel void @add_inline_imm_0.0_f16(ptr addrspace(1) %out, half %x
;
; GFX11-TRUE16-LABEL: add_inline_imm_0.0_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, 0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x00,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -795,11 +791,9 @@ define amdgpu_kernel void @add_inline_imm_0.0_f16(ptr addrspace(1) %out, half %x
;
; GFX11-FAKE16-LABEL: add_inline_imm_0.0_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, 0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x00,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -807,26 +801,25 @@ define amdgpu_kernel void @add_inline_imm_0.0_f16(ptr addrspace(1) %out, half %x
;
; VI-LABEL: add_inline_imm_0.0_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, 0 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0x00,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, 0 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0x00,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_0.0_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, 0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, 0.0
@@ -837,11 +830,9 @@ define amdgpu_kernel void @add_inline_imm_0.0_f16(ptr addrspace(1) %out, half %x
define amdgpu_kernel void @add_inline_imm_0.5_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_0.5_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, 0.5 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe0,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -849,11 +840,9 @@ define amdgpu_kernel void @add_inline_imm_0.5_f16(ptr addrspace(1) %out, half %x
;
; GFX11-TRUE16-LABEL: add_inline_imm_0.5_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, 0.5 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe0,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -861,11 +850,9 @@ define amdgpu_kernel void @add_inline_imm_0.5_f16(ptr addrspace(1) %out, half %x
;
; GFX11-FAKE16-LABEL: add_inline_imm_0.5_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, 0.5 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe0,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -873,26 +860,25 @@ define amdgpu_kernel void @add_inline_imm_0.5_f16(ptr addrspace(1) %out, half %x
;
; VI-LABEL: add_inline_imm_0.5_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, 0.5 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0xe0,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, 0.5 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0xe0,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_0.5_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, 0.5, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, 0.5
@@ -903,11 +889,9 @@ define amdgpu_kernel void @add_inline_imm_0.5_f16(ptr addrspace(1) %out, half %x
define amdgpu_kernel void @add_inline_imm_neg_0.5_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_neg_0.5_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, -0.5 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe2,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -915,11 +899,9 @@ define amdgpu_kernel void @add_inline_imm_neg_0.5_f16(ptr addrspace(1) %out, hal
;
; GFX11-TRUE16-LABEL: add_inline_imm_neg_0.5_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, -0.5 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe2,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -927,11 +909,9 @@ define amdgpu_kernel void @add_inline_imm_neg_0.5_f16(ptr addrspace(1) %out, hal
;
; GFX11-FAKE16-LABEL: add_inline_imm_neg_0.5_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, -0.5 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe2,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -939,26 +919,25 @@ define amdgpu_kernel void @add_inline_imm_neg_0.5_f16(ptr addrspace(1) %out, hal
;
; VI-LABEL: add_inline_imm_neg_0.5_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, -0.5 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0xe2,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, -0.5 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0xe2,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_neg_0.5_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, -0.5, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, -0.5
@@ -969,11 +948,9 @@ define amdgpu_kernel void @add_inline_imm_neg_0.5_f16(ptr addrspace(1) %out, hal
define amdgpu_kernel void @add_inline_imm_1.0_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_1.0_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, 1.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe4,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -981,11 +958,9 @@ define amdgpu_kernel void @add_inline_imm_1.0_f16(ptr addrspace(1) %out, half %x
;
; GFX11-TRUE16-LABEL: add_inline_imm_1.0_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, 1.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe4,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -993,11 +968,9 @@ define amdgpu_kernel void @add_inline_imm_1.0_f16(ptr addrspace(1) %out, half %x
;
; GFX11-FAKE16-LABEL: add_inline_imm_1.0_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, 1.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe4,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1005,26 +978,25 @@ define amdgpu_kernel void @add_inline_imm_1.0_f16(ptr addrspace(1) %out, half %x
;
; VI-LABEL: add_inline_imm_1.0_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, 1.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0xe4,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, 1.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0xe4,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_1.0_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, 1.0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, 1.0
@@ -1035,11 +1007,9 @@ define amdgpu_kernel void @add_inline_imm_1.0_f16(ptr addrspace(1) %out, half %x
define amdgpu_kernel void @add_inline_imm_neg_1.0_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_neg_1.0_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, -1.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe6,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -1047,11 +1017,9 @@ define amdgpu_kernel void @add_inline_imm_neg_1.0_f16(ptr addrspace(1) %out, hal
;
; GFX11-TRUE16-LABEL: add_inline_imm_neg_1.0_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, -1.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe6,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1059,11 +1027,9 @@ define amdgpu_kernel void @add_inline_imm_neg_1.0_f16(ptr addrspace(1) %out, hal
;
; GFX11-FAKE16-LABEL: add_inline_imm_neg_1.0_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, -1.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe6,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1071,26 +1037,25 @@ define amdgpu_kernel void @add_inline_imm_neg_1.0_f16(ptr addrspace(1) %out, hal
;
; VI-LABEL: add_inline_imm_neg_1.0_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, -1.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0xe6,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, -1.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0xe6,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_neg_1.0_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, -1.0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, -1.0
@@ -1101,11 +1066,9 @@ define amdgpu_kernel void @add_inline_imm_neg_1.0_f16(ptr addrspace(1) %out, hal
define amdgpu_kernel void @add_inline_imm_2.0_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_2.0_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, 2.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe8,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -1113,11 +1076,9 @@ define amdgpu_kernel void @add_inline_imm_2.0_f16(ptr addrspace(1) %out, half %x
;
; GFX11-TRUE16-LABEL: add_inline_imm_2.0_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, 2.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe8,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1125,11 +1086,9 @@ define amdgpu_kernel void @add_inline_imm_2.0_f16(ptr addrspace(1) %out, half %x
;
; GFX11-FAKE16-LABEL: add_inline_imm_2.0_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, 2.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xe8,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1137,26 +1096,25 @@ define amdgpu_kernel void @add_inline_imm_2.0_f16(ptr addrspace(1) %out, half %x
;
; VI-LABEL: add_inline_imm_2.0_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, 2.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0xe8,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, 2.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0xe8,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_2.0_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, 2.0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, 2.0
@@ -1167,11 +1125,9 @@ define amdgpu_kernel void @add_inline_imm_2.0_f16(ptr addrspace(1) %out, half %x
define amdgpu_kernel void @add_inline_imm_neg_2.0_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_neg_2.0_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, -2.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xea,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -1179,11 +1135,9 @@ define amdgpu_kernel void @add_inline_imm_neg_2.0_f16(ptr addrspace(1) %out, hal
;
; GFX11-TRUE16-LABEL: add_inline_imm_neg_2.0_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, -2.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xea,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1191,11 +1145,9 @@ define amdgpu_kernel void @add_inline_imm_neg_2.0_f16(ptr addrspace(1) %out, hal
;
; GFX11-FAKE16-LABEL: add_inline_imm_neg_2.0_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, -2.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xea,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1203,26 +1155,25 @@ define amdgpu_kernel void @add_inline_imm_neg_2.0_f16(ptr addrspace(1) %out, hal
;
; VI-LABEL: add_inline_imm_neg_2.0_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, -2.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0xea,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, -2.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0xea,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_neg_2.0_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, -2.0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, -2.0
@@ -1233,11 +1184,9 @@ define amdgpu_kernel void @add_inline_imm_neg_2.0_f16(ptr addrspace(1) %out, hal
define amdgpu_kernel void @add_inline_imm_4.0_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_4.0_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, 4.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xec,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -1245,11 +1194,9 @@ define amdgpu_kernel void @add_inline_imm_4.0_f16(ptr addrspace(1) %out, half %x
;
; GFX11-TRUE16-LABEL: add_inline_imm_4.0_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, 4.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xec,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1257,11 +1204,9 @@ define amdgpu_kernel void @add_inline_imm_4.0_f16(ptr addrspace(1) %out, half %x
;
; GFX11-FAKE16-LABEL: add_inline_imm_4.0_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, 4.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xec,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1269,26 +1214,25 @@ define amdgpu_kernel void @add_inline_imm_4.0_f16(ptr addrspace(1) %out, half %x
;
; VI-LABEL: add_inline_imm_4.0_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, 4.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0xec,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, 4.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0xec,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_4.0_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, 4.0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, 4.0
@@ -1299,11 +1243,9 @@ define amdgpu_kernel void @add_inline_imm_4.0_f16(ptr addrspace(1) %out, half %x
define amdgpu_kernel void @add_inline_imm_neg_4.0_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_neg_4.0_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, -4.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xee,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -1311,11 +1253,9 @@ define amdgpu_kernel void @add_inline_imm_neg_4.0_f16(ptr addrspace(1) %out, hal
;
; GFX11-TRUE16-LABEL: add_inline_imm_neg_4.0_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, -4.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xee,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1323,11 +1263,9 @@ define amdgpu_kernel void @add_inline_imm_neg_4.0_f16(ptr addrspace(1) %out, hal
;
; GFX11-FAKE16-LABEL: add_inline_imm_neg_4.0_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, -4.0 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0xee,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1335,26 +1273,25 @@ define amdgpu_kernel void @add_inline_imm_neg_4.0_f16(ptr addrspace(1) %out, hal
;
; VI-LABEL: add_inline_imm_neg_4.0_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, -4.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0xee,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, -4.0 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0xee,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_neg_4.0_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, -4.0, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, -4.0
@@ -1561,11 +1498,9 @@ define amdgpu_kernel void @commute_add_literal_f16(ptr addrspace(1) %out, ptr ad
define amdgpu_kernel void @add_inline_imm_1_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_1_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, 1 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x02,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -1573,11 +1508,9 @@ define amdgpu_kernel void @add_inline_imm_1_f16(ptr addrspace(1) %out, half %x)
;
; GFX11-TRUE16-LABEL: add_inline_imm_1_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, 1 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x02,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1585,11 +1518,9 @@ define amdgpu_kernel void @add_inline_imm_1_f16(ptr addrspace(1) %out, half %x)
;
; GFX11-FAKE16-LABEL: add_inline_imm_1_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, 1 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x02,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1597,26 +1528,25 @@ define amdgpu_kernel void @add_inline_imm_1_f16(ptr addrspace(1) %out, half %x)
;
; VI-LABEL: add_inline_imm_1_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, 1 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0x02,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, 1 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0x02,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_1_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, 0x33800000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, 0xH0001
@@ -1627,11 +1557,9 @@ define amdgpu_kernel void @add_inline_imm_1_f16(ptr addrspace(1) %out, half %x)
define amdgpu_kernel void @add_inline_imm_2_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_2_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, 2 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x04,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -1639,11 +1567,9 @@ define amdgpu_kernel void @add_inline_imm_2_f16(ptr addrspace(1) %out, half %x)
;
; GFX11-TRUE16-LABEL: add_inline_imm_2_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, 2 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x04,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1651,11 +1577,9 @@ define amdgpu_kernel void @add_inline_imm_2_f16(ptr addrspace(1) %out, half %x)
;
; GFX11-FAKE16-LABEL: add_inline_imm_2_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, 2 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x04,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1663,26 +1587,25 @@ define amdgpu_kernel void @add_inline_imm_2_f16(ptr addrspace(1) %out, half %x)
;
; VI-LABEL: add_inline_imm_2_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, 2 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0x04,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, 2 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0x04,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_2_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, 0x34000000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, 0xH0002
@@ -1693,11 +1616,9 @@ define amdgpu_kernel void @add_inline_imm_2_f16(ptr addrspace(1) %out, half %x)
define amdgpu_kernel void @add_inline_imm_16_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_16_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, 16 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x20,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -1705,11 +1626,9 @@ define amdgpu_kernel void @add_inline_imm_16_f16(ptr addrspace(1) %out, half %x)
;
; GFX11-TRUE16-LABEL: add_inline_imm_16_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, 16 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x20,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1717,11 +1636,9 @@ define amdgpu_kernel void @add_inline_imm_16_f16(ptr addrspace(1) %out, half %x)
;
; GFX11-FAKE16-LABEL: add_inline_imm_16_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, 16 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x20,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -1729,26 +1646,25 @@ define amdgpu_kernel void @add_inline_imm_16_f16(ptr addrspace(1) %out, half %x)
;
; VI-LABEL: add_inline_imm_16_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, 16 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0x20,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, 16 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0x20,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_16_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, 0x35800000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, 0xH0010
@@ -1996,11 +1912,9 @@ define amdgpu_kernel void @add_inline_imm_neg_16_f16(ptr addrspace(1) %out, ptr
define amdgpu_kernel void @add_inline_imm_63_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_63_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, 63 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x7e,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -2008,11 +1922,9 @@ define amdgpu_kernel void @add_inline_imm_63_f16(ptr addrspace(1) %out, half %x)
;
; GFX11-TRUE16-LABEL: add_inline_imm_63_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, 63 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x7e,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -2020,11 +1932,9 @@ define amdgpu_kernel void @add_inline_imm_63_f16(ptr addrspace(1) %out, half %x)
;
; GFX11-FAKE16-LABEL: add_inline_imm_63_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, 63 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x7e,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -2032,26 +1942,25 @@ define amdgpu_kernel void @add_inline_imm_63_f16(ptr addrspace(1) %out, half %x)
;
; VI-LABEL: add_inline_imm_63_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, 63 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0x7e,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, 63 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0x7e,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_63_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, 0x367c0000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, 0xH003F
@@ -2062,11 +1971,9 @@ define amdgpu_kernel void @add_inline_imm_63_f16(ptr addrspace(1) %out, half %x)
define amdgpu_kernel void @add_inline_imm_64_f16(ptr addrspace(1) %out, half %x) {
; GFX10-LABEL: add_inline_imm_64_f16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0xa1,0xbf]
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8 ; encoding: [0x84,0x00,0x00,0xf4,0x08,0x00,0x00,0xfa]
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x04,0xf4,0x00,0x00,0x00,0xfa]
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x08,0xf4,0x00,0x00,0x00,0xfa]
; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0xc0,0x8c,0xbf]
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x03,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX10-NEXT: v_add_f16_e64 v0, s2, 64 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x80,0x01,0x02]
; GFX10-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x03,0x82,0xbe]
; GFX10-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
@@ -2074,11 +1981,9 @@ define amdgpu_kernel void @add_inline_imm_64_f16(ptr addrspace(1) %out, half %x)
;
; GFX11-TRUE16-LABEL: add_inline_imm_64_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-TRUE16-NEXT: v_add_f16_e64 v0.l, s2, 64 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x80,0x01,0x02]
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -2086,11 +1991,9 @@ define amdgpu_kernel void @add_inline_imm_64_f16(ptr addrspace(1) %out, half %x)
;
; GFX11-FAKE16-LABEL: add_inline_imm_64_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1 ; encoding: [0x01,0x00,0x85,0xbf]
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x8 ; encoding: [0x82,0x00,0x00,0xf4,0x08,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 ; encoding: [0x02,0x00,0x04,0xf4,0x00,0x00,0x00,0xf8]
-; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 ; encoding: [0x02,0x00,0x08,0xf4,0x00,0x00,0x00,0xf8]
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x07,0xfc,0x89,0xbf]
+; GFX11-FAKE16-NEXT: s_mov_b32 s3, 0x31016000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0x60,0x01,0x31]
; GFX11-FAKE16-NEXT: v_add_f16_e64 v0, s2, 64 ; encoding: [0x00,0x00,0x32,0xd5,0x02,0x80,0x01,0x02]
; GFX11-FAKE16-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
; GFX11-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x64,0xe0,0x00,0x00,0x00,0x80]
@@ -2098,26 +2001,25 @@ define amdgpu_kernel void @add_inline_imm_64_f16(ptr addrspace(1) %out, half %x)
;
; VI-LABEL: add_inline_imm_64_f16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s4, s[8:9], 0x8 ; encoding: [0x04,0x01,0x02,0xc0,0x08,0x00,0x00,0x00]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0 ; encoding: [0x04,0x00,0x06,0xc0,0x00,0x00,0x00,0x00]
-; VI-NEXT: s_mov_b32 s3, 0x1100f000 ; encoding: [0xff,0x00,0x83,0xbe,0x00,0xf0,0x00,0x11]
-; VI-NEXT: s_mov_b32 s2, -1 ; encoding: [0xc1,0x00,0x82,0xbe]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0 ; encoding: [0x04,0x00,0x0a,0xc0,0x00,0x00,0x00,0x00]
+; VI-NEXT: s_mov_b32 s7, 0x1100f000 ; encoding: [0xff,0x00,0x87,0xbe,0x00,0xf0,0x00,0x11]
+; VI-NEXT: s_mov_b32 s6, -1 ; encoding: [0xc1,0x00,0x86,0xbe]
; VI-NEXT: s_waitcnt lgkmcnt(0) ; encoding: [0x7f,0x00,0x8c,0xbf]
-; VI-NEXT: v_add_f16_e64 v0, s4, 64 ; encoding: [0x00,0x00,0x1f,0xd1,0x04,0x80,0x01,0x00]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x00,0x80]
+; VI-NEXT: s_mov_b32 s4, s0 ; encoding: [0x00,0x00,0x84,0xbe]
+; VI-NEXT: s_mov_b32 s5, s1 ; encoding: [0x01,0x00,0x85,0xbe]
+; VI-NEXT: v_add_f16_e64 v0, s2, 64 ; encoding: [0x00,0x00,0x1f,0xd1,0x02,0x80,0x01,0x00]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 ; encoding: [0x00,0x00,0x68,0xe0,0x00,0x00,0x01,0x80]
; VI-NEXT: s_endpgm ; encoding: [0x00,0x00,0x81,0xbf]
;
; SI-LABEL: add_inline_imm_64_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_f16_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f32_f16_e32 v0, s0
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: v_add_f32_e32 v0, 0x36800000, v0
; SI-NEXT: v_cvt_f16_f32_e32 v0, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
%y = fadd half %x, 0xH0040
diff --git a/llvm/test/CodeGen/AMDGPU/immv216.ll b/llvm/test/CodeGen/AMDGPU/immv216.ll
index ae5199a3f7906..233535e3d33e8 100644
--- a/llvm/test/CodeGen/AMDGPU/immv216.ll
+++ b/llvm/test/CodeGen/AMDGPU/immv216.ll
@@ -118,18 +118,18 @@ define amdgpu_kernel void @store_literal_imm_v2f16(ptr addrspace(1) %out) #0 {
}
; GCN-LABEL: {{^}}add_inline_imm_0.0_v2f16:
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 0 ; encoding
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 0 ; encoding
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONST0:v[0-9]+]], 0
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONST0]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], 0
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, 0
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_0.0_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -139,22 +139,22 @@ define amdgpu_kernel void @add_inline_imm_0.0_v2f16(ptr addrspace(1) %out, <2 x
}
; GCN-LABEL: {{^}}add_inline_imm_0.5_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 0.5 op_sel_hi:[1,0] ; encoding: [0x00,0x40,0x0f,0xcc,0x02,0xe0,0x01,0x0a]
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 0.5 op_sel_hi:[1,0] ; encoding: [0x00,0x40,0x0f,0xcc,0x02,0xe0,0x01,0x0a]
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 0.5 op_sel_hi:[1,0] ; encoding: [0x00,0x40,0x8f,0xd3,0x06,0xe0,0x01,0x08]
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 0.5 op_sel_hi:[1,0] ; encoding: [0x00,0x40,0x8f,0xd3,0x02,0xe0,0x01,0x08]
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONST05:v[0-9]+]], 0x3800
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONST05]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], 0.5
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, 0.5
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_0.5_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -164,22 +164,22 @@ define amdgpu_kernel void @add_inline_imm_0.5_v2f16(ptr addrspace(1) %out, <2 x
}
; GCN-LABEL: {{^}}add_inline_imm_neg_0.5_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], -0.5 op_sel_hi:[1,0] ; encoding: [0x00,0x40,0x0f,0xcc,0x02,0xe2,0x01,0x0a]
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, -0.5 op_sel_hi:[1,0] ; encoding: [0x00,0x40,0x0f,0xcc,0x02,0xe2,0x01,0x0a]
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], -0.5 op_sel_hi:[1,0] ; encoding: [0x00,0x40,0x8f,0xd3,0x06,0xe2,0x01,0x08]
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, -0.5 op_sel_hi:[1,0] ; encoding: [0x00,0x40,0x8f,0xd3,0x02,0xe2,0x01,0x08]
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONSTM05:v[0-9]+]], 0xb800
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONSTM05]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], -0.5
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, -0.5
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_neg_0.5_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -189,22 +189,22 @@ define amdgpu_kernel void @add_inline_imm_neg_0.5_v2f16(ptr addrspace(1) %out, <
}
; GCN-LABEL: {{^}}add_inline_imm_1.0_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 1.0 op_sel_hi:[1,0] ; encoding:
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 1.0 op_sel_hi:[1,0] ; encoding:
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 1.0 op_sel_hi:[1,0] ; encoding
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 1.0 op_sel_hi:[1,0] ; encoding
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONST1:v[0-9]+]], 0x3c00
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONST1]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], 1.0
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, 1.0
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_1.0_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -214,23 +214,23 @@ define amdgpu_kernel void @add_inline_imm_1.0_v2f16(ptr addrspace(1) %out, <2 x
}
; GCN-LABEL: {{^}}add_inline_imm_neg_1.0_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], -1.0 op_sel_hi:[1,0] ; encoding
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, -1.0 op_sel_hi:[1,0] ; encoding
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], -1.0 op_sel_hi:[1,0] ; encoding
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, -1.0 op_sel_hi:[1,0] ; encoding
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONST1:v[0-9]+]], 0xbc00
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONST1]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], -1.0
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, -1.0
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_neg_1.0_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -240,22 +240,22 @@ define amdgpu_kernel void @add_inline_imm_neg_1.0_v2f16(ptr addrspace(1) %out, <
}
; GCN-LABEL: {{^}}add_inline_imm_2.0_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 2.0 op_sel_hi:[1,0] ; encoding
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 2.0 op_sel_hi:[1,0] ; encoding
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 2.0 op_sel_hi:[1,0] ; encoding
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 2.0 op_sel_hi:[1,0] ; encoding
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONST2:v[0-9]+]], 0x4000
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONST2]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], 2.0
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, 2.0
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_2.0_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -265,22 +265,22 @@ define amdgpu_kernel void @add_inline_imm_2.0_v2f16(ptr addrspace(1) %out, <2 x
}
; GCN-LABEL: {{^}}add_inline_imm_neg_2.0_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], -2.0 op_sel_hi:[1,0] ; encoding
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, -2.0 op_sel_hi:[1,0] ; encoding
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], -2.0 op_sel_hi:[1,0] ; encoding
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, -2.0 op_sel_hi:[1,0] ; encoding
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONSTM2:v[0-9]+]], 0xc000
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONSTM2]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], -2.0
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, -2.0
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_neg_2.0_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -290,22 +290,22 @@ define amdgpu_kernel void @add_inline_imm_neg_2.0_v2f16(ptr addrspace(1) %out, <
}
; GCN-LABEL: {{^}}add_inline_imm_4.0_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 4.0 op_sel_hi:[1,0] ; encoding
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 4.0 op_sel_hi:[1,0] ; encoding
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 4.0 op_sel_hi:[1,0] ; encoding
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 4.0 op_sel_hi:[1,0] ; encoding
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONST4:v[0-9]+]], 0x4400
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONST4]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], 4.0
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, 4.0
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_4.0_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -315,22 +315,22 @@ define amdgpu_kernel void @add_inline_imm_4.0_v2f16(ptr addrspace(1) %out, <2 x
}
; GCN-LABEL: {{^}}add_inline_imm_neg_4.0_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], -4.0 op_sel_hi:[1,0] ; encoding
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, -4.0 op_sel_hi:[1,0] ; encoding
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], -4.0 op_sel_hi:[1,0] ; encoding
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, -4.0 op_sel_hi:[1,0] ; encoding
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONSTM4:v[0-9]+]], 0xc400
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONSTM4]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], -4.0
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, -4.0
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_neg_4.0_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -365,8 +365,8 @@ define amdgpu_kernel void @commute_add_inline_imm_0.5_v2f16(ptr addrspace(1) %ou
; GCN-LABEL: {{^}}commute_add_literal_v2f16:
; GFX10: v_pk_add_f16 v0, 0x6400, v0 op_sel_hi:[0,1] ; encoding: [0x00,0x40,0x0f,0xcc,0xff,0x00,0x02,0x12,0x00,0x64,0x00,0x00]
-; GFX9-DAG: buffer_load_dword [[VAL:v[0-9]+]]
-; GFX9-DAG: s_movk_i32 [[K:s[0-9]+]], 0x6400 ; encoding
+; GFX9: buffer_load_dword [[VAL:v[0-9]+]]
+; GFX9: s_movk_i32 [[K:s[0-9]+]], 0x6400 ; encoding
; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], [[K]] op_sel_hi:[1,0] ; encoding: [0x00,0x40,0x8f,0xd3,0x00,0x01,0x00,0x08]
; GFX9: buffer_store_dword [[REG]]
@@ -386,22 +386,22 @@ define amdgpu_kernel void @commute_add_literal_v2f16(ptr addrspace(1) %out, ptr
}
; GCN-LABEL: {{^}}add_inline_imm_1_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 1 op_sel_hi:[1,0] ; encoding
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 1 op_sel_hi:[1,0] ; encoding
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 1 op_sel_hi:[1,0] ; encoding
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 1 op_sel_hi:[1,0] ; encoding
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONST1:v[0-9]+]], 1 ; encoding
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONST1]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], 1 ; encoding
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, 1 ; encoding
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_1_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -411,23 +411,23 @@ define amdgpu_kernel void @add_inline_imm_1_v2f16(ptr addrspace(1) %out, <2 x ha
}
; GCN-LABEL: {{^}}add_inline_imm_2_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 2 op_sel_hi:[1,0] ; encoding
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 2 op_sel_hi:[1,0] ; encoding
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 2 op_sel_hi:[1,0] ; encoding
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 2 op_sel_hi:[1,0] ; encoding
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONST2:v[0-9]+]], 2 ; encoding
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONST2]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], 2 ; encoding
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, 2 ; encoding
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_2_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -437,23 +437,23 @@ define amdgpu_kernel void @add_inline_imm_2_v2f16(ptr addrspace(1) %out, <2 x ha
}
; GCN-LABEL: {{^}}add_inline_imm_16_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 16 op_sel_hi:[1,0] ; encoding
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 16 op_sel_hi:[1,0] ; encoding
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 16 op_sel_hi:[1,0] ; encoding
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 16 op_sel_hi:[1,0] ; encoding
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONST16:v[0-9]+]], 16 ; encoding
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONST16]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], 16 ; encoding
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, 16 ; encoding
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_16_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -467,12 +467,12 @@ define amdgpu_kernel void @add_inline_imm_16_v2f16(ptr addrspace(1) %out, <2 x h
; GFX10: v_mov_b32_e32 [[REG:v[0-9]+]], [[VAL]]
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_add_i32 [[VAL:s[0-9]+]], s6, -1
+; GFX9: s_add_i32 [[VAL:s[0-9]+]], s{{[0-9]+}}, -1
; GFX9: v_mov_b32_e32 [[REG:v[0-9]+]], [[VAL]]
; GFX9: buffer_store_dword [[REG]]
-; VI: s_load_dword [[VAL:s[0-9]+]]
-; VI: s_add_i32 [[ADD:s[0-9]+]], [[VAL]], -1 ; encoding
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; VI: s_add_i32 [[ADD:s[0-9]+]], s{{[0-9]+}}, -1 ; encoding
; VI: v_mov_b32_e32 [[REG:v[0-9]+]], [[ADD]]
; VI: buffer_store_dword [[REG]]
define amdgpu_kernel void @add_inline_imm_neg_1_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -488,12 +488,12 @@ define amdgpu_kernel void @add_inline_imm_neg_1_v2f16(ptr addrspace(1) %out, <2
; GFX10: v_mov_b32_e32 [[REG:v[0-9]+]], [[VAL]]
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_add_i32 [[VAL:s[0-9]+]], s6, 0xfffefffe
+; GFX9: s_add_i32 [[VAL:s[0-9]+]], s{{[0-9]+}}, 0xfffefffe
; GFX9: v_mov_b32_e32 [[REG:v[0-9]+]], [[VAL]]
; GFX9: buffer_store_dword [[REG]]
-; VI: s_load_dword [[VAL:s[0-9]+]]
-; VI: s_add_i32 [[ADD:s[0-9]+]], [[VAL]], 0xfffefffe ; encoding
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; VI: s_add_i32 [[ADD:s[0-9]+]], s{{[0-9]+}}, 0xfffefffe ; encoding
; VI: v_mov_b32_e32 [[REG:v[0-9]+]], [[ADD]]
; VI: buffer_store_dword [[REG]]
define amdgpu_kernel void @add_inline_imm_neg_2_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -509,13 +509,13 @@ define amdgpu_kernel void @add_inline_imm_neg_2_v2f16(ptr addrspace(1) %out, <2
; GFX10: v_mov_b32_e32 [[REG:v[0-9]+]], [[VAL]]
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_add_i32 [[VAL:s[0-9]+]], s6, 0xfff0fff0
+; GFX9: s_add_i32 [[VAL:s[0-9]+]], s{{[0-9]+}}, 0xfff0fff0
; GFX9: v_mov_b32_e32 [[REG:v[0-9]+]], [[VAL]]
; GFX9: buffer_store_dword [[REG]]
-; VI: s_load_dword [[VAL:s[0-9]+]]
-; VI: s_add_i32 [[ADD:s[0-9]+]], [[VAL]], 0xfff0fff0 ; encoding
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; VI: s_add_i32 [[ADD:s[0-9]+]], s{{[0-9]+}}, 0xfff0fff0 ; encoding
; VI: v_mov_b32_e32 [[REG:v[0-9]+]], [[ADD]]
; VI: buffer_store_dword [[REG]]
define amdgpu_kernel void @add_inline_imm_neg_16_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -527,22 +527,22 @@ define amdgpu_kernel void @add_inline_imm_neg_16_v2f16(ptr addrspace(1) %out, <2
}
; GCN-LABEL: {{^}}add_inline_imm_63_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 63
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 63
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 63
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 63
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONST63:v[0-9]+]], 63
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONST63]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], 63
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, 63
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_63_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
@@ -552,22 +552,22 @@ define amdgpu_kernel void @add_inline_imm_63_v2f16(ptr addrspace(1) %out, <2 x h
}
; GCN-LABEL: {{^}}add_inline_imm_64_v2f16:
-; GFX10: s_load_{{dword|b32}} [[VAL:s[0-9]+]]
-; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 64
+; GFX10: s_load_{{b128|dwordx4}} s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX10: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 64
; GFX10: buffer_store_{{dword|b32}} [[REG]]
-; GFX9: s_load_dword [[VAL:s[0-9]+]]
-; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], [[VAL]], 64
+; GFX9: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
+; GFX9: v_pk_add_f16 [[REG:v[0-9]+]], s{{[0-9]+}}, 64
; GFX9: buffer_store_dword [[REG]]
; FIXME: Shouldn't need right shift and SDWA, also extra copy
-; VI-DAG: s_load_dword [[VAL:s[0-9]+]]
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x0
; VI-DAG: v_mov_b32_e32 [[CONST64:v[0-9]+]], 64
-; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], [[VAL]], 16
+; VI-DAG: s_lshr_b32 [[SHR:s[0-9]+]], s{{[0-9]+}}, 16
; VI-DAG: v_mov_b32_e32 [[V_SHR:v[0-9]+]], [[SHR]]
; VI-DAG: v_add_f16_sdwa v{{[0-9]+}}, [[V_SHR]], [[CONST64]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, [[VAL]], 64
+; VI-DAG: v_add_f16_e64 v{{[0-9]+}}, s{{[0-9]+}}, 64
; VI: v_or_b32
; VI: buffer_store_dword
define amdgpu_kernel void @add_inline_imm_64_v2f16(ptr addrspace(1) %out, <2 x half> %x) #0 {
diff --git a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
index 1961ba53c1661..decb89937ee96 100644
--- a/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
+++ b/llvm/test/CodeGen/AMDGPU/indirect-addressing-si.ll
@@ -11,8 +11,7 @@
define amdgpu_kernel void @extract_w_offset(ptr addrspace(1) %out, i32 %in) {
; GENERIC-LABEL: extract_w_offset:
; GENERIC: ; %bb.0: ; %entry
-; GENERIC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GENERIC-NEXT: s_load_dword s4, s[4:5], 0xb
+; GENERIC-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GENERIC-NEXT: s_mov_b32 s3, 0xf000
; GENERIC-NEXT: s_mov_b32 s2, -1
; GENERIC-NEXT: v_mov_b32_e32 v0, 0x40400000
@@ -29,7 +28,9 @@ define amdgpu_kernel void @extract_w_offset(ptr addrspace(1) %out, i32 %in) {
; GENERIC-NEXT: v_mov_b32_e32 v11, 0x41700000
; GENERIC-NEXT: v_mov_b32_e32 v12, 0x41800000
; GENERIC-NEXT: s_waitcnt lgkmcnt(0)
-; GENERIC-NEXT: s_add_i32 s6, s4, 1
+; GENERIC-NEXT: s_mov_b32 s0, s4
+; GENERIC-NEXT: s_mov_b32 s1, s5
+; GENERIC-NEXT: s_add_i32 s6, s6, 1
; GENERIC-NEXT: s_cmp_eq_u32 s6, 1
; GENERIC-NEXT: s_cselect_b64 s[4:5], -1, 0
; GENERIC-NEXT: v_cndmask_b32_e64 v13, 1.0, 2.0, s[4:5]
@@ -152,14 +153,15 @@ define amdgpu_kernel void @extract_w_offset(ptr addrspace(1) %out, i32 %in) {
;
; SI-MOVREL-LABEL: extract_w_offset:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
-; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
-; SI-MOVREL-NEXT: s_mov_b32 s2, -1
-; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-MOVREL-NEXT: s_add_i32 m0, s6, 1
+; SI-MOVREL-NEXT: s_mov_b32 s7, 0xf000
+; SI-MOVREL-NEXT: s_mov_b32 s6, -1
; SI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
+; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-MOVREL-NEXT: s_add_i32 m0, s2, 1
+; SI-MOVREL-NEXT: s_mov_b32 s4, s0
+; SI-MOVREL-NEXT: s_mov_b32 s5, s1
; SI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
; SI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
@@ -175,19 +177,20 @@ define amdgpu_kernel void @extract_w_offset(ptr addrspace(1) %out, i32 %in) {
; SI-MOVREL-NEXT: v_mov_b32_e32 v14, 0x41700000
; SI-MOVREL-NEXT: v_mov_b32_e32 v15, 0x41800000
; SI-MOVREL-NEXT: v_movrels_b32_e32 v0, v0
-; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-MOVREL-NEXT: s_endpgm
;
; VI-MOVREL-LABEL: extract_w_offset:
; VI-MOVREL: ; %bb.0: ; %entry
-; VI-MOVREL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; VI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
; VI-MOVREL-NEXT: s_add_i32 m0, s2, 1
-; VI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v16, s0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v17, s1
; VI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
; VI-MOVREL-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-MOVREL-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -200,22 +203,21 @@ define amdgpu_kernel void @extract_w_offset(ptr addrspace(1) %out, i32 %in) {
; VI-MOVREL-NEXT: v_mov_b32_e32 v13, 0x41600000
; VI-MOVREL-NEXT: v_mov_b32_e32 v14, 0x41700000
; VI-MOVREL-NEXT: v_mov_b32_e32 v15, 0x41800000
-; VI-MOVREL-NEXT: v_movrels_b32_e32 v2, v0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v0, s0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v1, s1
-; VI-MOVREL-NEXT: flat_store_dword v[0:1], v2
+; VI-MOVREL-NEXT: v_movrels_b32_e32 v0, v0
+; VI-MOVREL-NEXT: flat_store_dword v[16:17], v0
; VI-MOVREL-NEXT: s_endpgm
;
; VI-IDXMODE-LABEL: extract_w_offset:
; VI-IDXMODE: ; %bb.0: ; %entry
-; VI-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
-; VI-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
-; VI-IDXMODE-NEXT: s_add_i32 s2, s2, 1
; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
+; VI-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v16, s0
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v17, s1
+; VI-IDXMODE-NEXT: s_add_i32 s0, s2, 1
; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, 0x40a00000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -228,24 +230,21 @@ define amdgpu_kernel void @extract_w_offset(ptr addrspace(1) %out, i32 %in) {
; VI-IDXMODE-NEXT: v_mov_b32_e32 v13, 0x41600000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v14, 0x41700000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v15, 0x41800000
-; VI-IDXMODE-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, v0
+; VI-IDXMODE-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, v0
; VI-IDXMODE-NEXT: s_set_gpr_idx_off
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, s0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, s1
-; VI-IDXMODE-NEXT: flat_store_dword v[0:1], v2
+; VI-IDXMODE-NEXT: flat_store_dword v[16:17], v0
; VI-IDXMODE-NEXT: s_endpgm
;
; GFX9-IDXMODE-LABEL: extract_w_offset:
; GFX9-IDXMODE: ; %bb.0: ; %entry
-; GFX9-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v16, 0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
; GFX9-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-IDXMODE-NEXT: s_add_i32 s2, s2, 1
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v4, 0x40a00000
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v5, 0x40c00000
@@ -275,61 +274,60 @@ entry:
define amdgpu_kernel void @extract_w_offset_salu_use_vector(ptr addrspace(1) %out, i32 %in, <16 x i32> %or.val) {
; GENERIC-LABEL: extract_w_offset_salu_use_vector:
; GENERIC: ; %bb.0: ; %entry
-; GENERIC-NEXT: s_load_dword s2, s[4:5], 0xb
-; GENERIC-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x19
-; GENERIC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GENERIC-NEXT: s_mov_b32 s3, 0xf000
+; GENERIC-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x9
+; GENERIC-NEXT: s_load_dwordx16 s[0:15], s[4:5], 0x19
; GENERIC-NEXT: s_waitcnt lgkmcnt(0)
-; GENERIC-NEXT: s_add_i32 s2, s2, 1
-; GENERIC-NEXT: s_or_b32 s4, s23, 16
-; GENERIC-NEXT: s_or_b32 s5, s22, 15
-; GENERIC-NEXT: s_or_b32 s6, s21, 14
-; GENERIC-NEXT: s_or_b32 s7, s20, 13
-; GENERIC-NEXT: s_or_b32 s19, s19, 12
-; GENERIC-NEXT: s_or_b32 s18, s18, 11
-; GENERIC-NEXT: s_or_b32 s17, s17, 10
-; GENERIC-NEXT: s_or_b32 s16, s16, 9
-; GENERIC-NEXT: s_or_b32 s15, s15, 8
-; GENERIC-NEXT: s_or_b32 s14, s14, 7
-; GENERIC-NEXT: s_or_b32 s13, s13, 6
-; GENERIC-NEXT: s_or_b32 s12, s12, 5
-; GENERIC-NEXT: s_or_b32 s11, s11, 4
-; GENERIC-NEXT: s_or_b32 s10, s10, 3
-; GENERIC-NEXT: s_or_b32 s8, s8, 1
-; GENERIC-NEXT: s_or_b32 s9, s9, 2
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 1
-; GENERIC-NEXT: s_cselect_b32 s8, s9, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 2
-; GENERIC-NEXT: s_cselect_b32 s8, s10, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 3
-; GENERIC-NEXT: s_cselect_b32 s8, s11, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 4
-; GENERIC-NEXT: s_cselect_b32 s8, s12, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 5
-; GENERIC-NEXT: s_cselect_b32 s8, s13, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 6
-; GENERIC-NEXT: s_cselect_b32 s8, s14, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 7
-; GENERIC-NEXT: s_cselect_b32 s8, s15, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 8
-; GENERIC-NEXT: s_cselect_b32 s8, s16, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 9
-; GENERIC-NEXT: s_cselect_b32 s8, s17, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 10
-; GENERIC-NEXT: s_cselect_b32 s8, s18, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 11
-; GENERIC-NEXT: s_cselect_b32 s8, s19, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 12
-; GENERIC-NEXT: s_cselect_b32 s7, s7, s8
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 13
-; GENERIC-NEXT: s_cselect_b32 s6, s6, s7
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 14
-; GENERIC-NEXT: s_cselect_b32 s5, s5, s6
-; GENERIC-NEXT: s_cmp_eq_u32 s2, 15
-; GENERIC-NEXT: s_cselect_b32 s4, s4, s5
-; GENERIC-NEXT: s_mov_b32 s2, -1
-; GENERIC-NEXT: v_mov_b32_e32 v0, s4
-; GENERIC-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GENERIC-NEXT: s_mov_b32 s19, 0xf000
+; GENERIC-NEXT: s_add_i32 s18, s18, 1
+; GENERIC-NEXT: s_or_b32 s15, s15, 16
+; GENERIC-NEXT: s_or_b32 s14, s14, 15
+; GENERIC-NEXT: s_or_b32 s13, s13, 14
+; GENERIC-NEXT: s_or_b32 s12, s12, 13
+; GENERIC-NEXT: s_or_b32 s11, s11, 12
+; GENERIC-NEXT: s_or_b32 s10, s10, 11
+; GENERIC-NEXT: s_or_b32 s9, s9, 10
+; GENERIC-NEXT: s_or_b32 s8, s8, 9
+; GENERIC-NEXT: s_or_b32 s7, s7, 8
+; GENERIC-NEXT: s_or_b32 s6, s6, 7
+; GENERIC-NEXT: s_or_b32 s5, s5, 6
+; GENERIC-NEXT: s_or_b32 s4, s4, 5
+; GENERIC-NEXT: s_or_b32 s3, s3, 4
+; GENERIC-NEXT: s_or_b32 s2, s2, 3
+; GENERIC-NEXT: s_or_b32 s0, s0, 1
+; GENERIC-NEXT: s_or_b32 s1, s1, 2
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 1
+; GENERIC-NEXT: s_cselect_b32 s0, s1, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 2
+; GENERIC-NEXT: s_cselect_b32 s0, s2, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 3
+; GENERIC-NEXT: s_cselect_b32 s0, s3, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 4
+; GENERIC-NEXT: s_cselect_b32 s0, s4, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 5
+; GENERIC-NEXT: s_cselect_b32 s0, s5, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 6
+; GENERIC-NEXT: s_cselect_b32 s0, s6, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 7
+; GENERIC-NEXT: s_cselect_b32 s0, s7, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 8
+; GENERIC-NEXT: s_cselect_b32 s0, s8, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 9
+; GENERIC-NEXT: s_cselect_b32 s0, s9, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 10
+; GENERIC-NEXT: s_cselect_b32 s0, s10, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 11
+; GENERIC-NEXT: s_cselect_b32 s0, s11, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 12
+; GENERIC-NEXT: s_cselect_b32 s0, s12, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 13
+; GENERIC-NEXT: s_cselect_b32 s0, s13, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 14
+; GENERIC-NEXT: s_cselect_b32 s0, s14, s0
+; GENERIC-NEXT: s_cmp_eq_u32 s18, 15
+; GENERIC-NEXT: s_cselect_b32 s0, s15, s0
+; GENERIC-NEXT: s_mov_b32 s18, -1
+; GENERIC-NEXT: v_mov_b32_e32 v0, s0
+; GENERIC-NEXT: buffer_store_dword v0, off, s[16:19], 0
; GENERIC-NEXT: s_endpgm
;
; NOOPT-LABEL: extract_w_offset_salu_use_vector:
@@ -440,146 +438,144 @@ define amdgpu_kernel void @extract_w_offset_salu_use_vector(ptr addrspace(1) %ou
;
; SI-MOVREL-LABEL: extract_w_offset_salu_use_vector:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-MOVREL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x19
-; SI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
-; SI-MOVREL-NEXT: s_mov_b32 s2, -1
+; SI-MOVREL-NEXT: s_load_dwordx4 s[20:23], s[4:5], 0x9
+; SI-MOVREL-NEXT: s_load_dwordx16 s[0:15], s[4:5], 0x19
+; SI-MOVREL-NEXT: s_mov_b32 s19, 0xf000
+; SI-MOVREL-NEXT: s_mov_b32 s18, -1
; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-MOVREL-NEXT: s_add_i32 m0, s6, 1
-; SI-MOVREL-NEXT: s_or_b32 s8, s8, 1
-; SI-MOVREL-NEXT: s_or_b32 s4, s23, 16
-; SI-MOVREL-NEXT: s_or_b32 s5, s22, 15
-; SI-MOVREL-NEXT: s_or_b32 s6, s21, 14
-; SI-MOVREL-NEXT: s_or_b32 s7, s20, 13
-; SI-MOVREL-NEXT: s_or_b32 s19, s19, 12
-; SI-MOVREL-NEXT: s_or_b32 s18, s18, 11
-; SI-MOVREL-NEXT: s_or_b32 s17, s17, 10
-; SI-MOVREL-NEXT: s_or_b32 s16, s16, 9
-; SI-MOVREL-NEXT: s_or_b32 s15, s15, 8
-; SI-MOVREL-NEXT: s_or_b32 s14, s14, 7
-; SI-MOVREL-NEXT: s_or_b32 s13, s13, 6
-; SI-MOVREL-NEXT: s_or_b32 s12, s12, 5
-; SI-MOVREL-NEXT: s_or_b32 s11, s11, 4
-; SI-MOVREL-NEXT: s_or_b32 s10, s10, 3
-; SI-MOVREL-NEXT: s_or_b32 s9, s9, 2
-; SI-MOVREL-NEXT: v_mov_b32_e32 v0, s8
-; SI-MOVREL-NEXT: v_mov_b32_e32 v1, s9
-; SI-MOVREL-NEXT: v_mov_b32_e32 v2, s10
-; SI-MOVREL-NEXT: v_mov_b32_e32 v3, s11
-; SI-MOVREL-NEXT: v_mov_b32_e32 v4, s12
-; SI-MOVREL-NEXT: v_mov_b32_e32 v5, s13
-; SI-MOVREL-NEXT: v_mov_b32_e32 v6, s14
-; SI-MOVREL-NEXT: v_mov_b32_e32 v7, s15
-; SI-MOVREL-NEXT: v_mov_b32_e32 v8, s16
-; SI-MOVREL-NEXT: v_mov_b32_e32 v9, s17
-; SI-MOVREL-NEXT: v_mov_b32_e32 v10, s18
-; SI-MOVREL-NEXT: v_mov_b32_e32 v11, s19
-; SI-MOVREL-NEXT: v_mov_b32_e32 v12, s7
-; SI-MOVREL-NEXT: v_mov_b32_e32 v13, s6
-; SI-MOVREL-NEXT: v_mov_b32_e32 v14, s5
-; SI-MOVREL-NEXT: v_mov_b32_e32 v15, s4
+; SI-MOVREL-NEXT: s_add_i32 m0, s22, 1
+; SI-MOVREL-NEXT: s_or_b32 s0, s0, 1
+; SI-MOVREL-NEXT: s_or_b32 s15, s15, 16
+; SI-MOVREL-NEXT: s_or_b32 s14, s14, 15
+; SI-MOVREL-NEXT: s_or_b32 s13, s13, 14
+; SI-MOVREL-NEXT: s_or_b32 s12, s12, 13
+; SI-MOVREL-NEXT: s_or_b32 s11, s11, 12
+; SI-MOVREL-NEXT: s_or_b32 s10, s10, 11
+; SI-MOVREL-NEXT: s_or_b32 s9, s9, 10
+; SI-MOVREL-NEXT: s_or_b32 s8, s8, 9
+; SI-MOVREL-NEXT: s_or_b32 s7, s7, 8
+; SI-MOVREL-NEXT: s_or_b32 s6, s6, 7
+; SI-MOVREL-NEXT: s_or_b32 s5, s5, 6
+; SI-MOVREL-NEXT: s_or_b32 s4, s4, 5
+; SI-MOVREL-NEXT: s_or_b32 s3, s3, 4
+; SI-MOVREL-NEXT: s_or_b32 s2, s2, 3
+; SI-MOVREL-NEXT: s_or_b32 s1, s1, 2
+; SI-MOVREL-NEXT: v_mov_b32_e32 v0, s0
+; SI-MOVREL-NEXT: s_mov_b32 s16, s20
+; SI-MOVREL-NEXT: s_mov_b32 s17, s21
+; SI-MOVREL-NEXT: v_mov_b32_e32 v1, s1
+; SI-MOVREL-NEXT: v_mov_b32_e32 v2, s2
+; SI-MOVREL-NEXT: v_mov_b32_e32 v3, s3
+; SI-MOVREL-NEXT: v_mov_b32_e32 v4, s4
+; SI-MOVREL-NEXT: v_mov_b32_e32 v5, s5
+; SI-MOVREL-NEXT: v_mov_b32_e32 v6, s6
+; SI-MOVREL-NEXT: v_mov_b32_e32 v7, s7
+; SI-MOVREL-NEXT: v_mov_b32_e32 v8, s8
+; SI-MOVREL-NEXT: v_mov_b32_e32 v9, s9
+; SI-MOVREL-NEXT: v_mov_b32_e32 v10, s10
+; SI-MOVREL-NEXT: v_mov_b32_e32 v11, s11
+; SI-MOVREL-NEXT: v_mov_b32_e32 v12, s12
+; SI-MOVREL-NEXT: v_mov_b32_e32 v13, s13
+; SI-MOVREL-NEXT: v_mov_b32_e32 v14, s14
+; SI-MOVREL-NEXT: v_mov_b32_e32 v15, s15
; SI-MOVREL-NEXT: v_movrels_b32_e32 v0, v0
-; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[16:19], 0
; SI-MOVREL-NEXT: s_endpgm
;
; VI-MOVREL-LABEL: extract_w_offset_salu_use_vector:
; VI-MOVREL: ; %bb.0: ; %entry
-; VI-MOVREL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-MOVREL-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-MOVREL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; VI-MOVREL-NEXT: s_load_dwordx16 s[0:15], s[4:5], 0x64
; VI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-MOVREL-NEXT: s_add_i32 m0, s2, 1
-; VI-MOVREL-NEXT: s_or_b32 s10, s10, 3
-; VI-MOVREL-NEXT: s_or_b32 s9, s9, 2
-; VI-MOVREL-NEXT: s_or_b32 s8, s8, 1
-; VI-MOVREL-NEXT: s_or_b32 s2, s23, 16
-; VI-MOVREL-NEXT: s_or_b32 s3, s22, 15
-; VI-MOVREL-NEXT: s_or_b32 s4, s21, 14
-; VI-MOVREL-NEXT: s_or_b32 s5, s20, 13
-; VI-MOVREL-NEXT: s_or_b32 s6, s19, 12
-; VI-MOVREL-NEXT: s_or_b32 s7, s18, 11
-; VI-MOVREL-NEXT: s_or_b32 s17, s17, 10
-; VI-MOVREL-NEXT: s_or_b32 s16, s16, 9
-; VI-MOVREL-NEXT: s_or_b32 s15, s15, 8
-; VI-MOVREL-NEXT: s_or_b32 s14, s14, 7
-; VI-MOVREL-NEXT: s_or_b32 s13, s13, 6
-; VI-MOVREL-NEXT: s_or_b32 s12, s12, 5
-; VI-MOVREL-NEXT: s_or_b32 s11, s11, 4
-; VI-MOVREL-NEXT: v_mov_b32_e32 v0, s8
-; VI-MOVREL-NEXT: v_mov_b32_e32 v1, s9
-; VI-MOVREL-NEXT: v_mov_b32_e32 v2, s10
-; VI-MOVREL-NEXT: v_mov_b32_e32 v3, s11
-; VI-MOVREL-NEXT: v_mov_b32_e32 v4, s12
-; VI-MOVREL-NEXT: v_mov_b32_e32 v5, s13
-; VI-MOVREL-NEXT: v_mov_b32_e32 v6, s14
-; VI-MOVREL-NEXT: v_mov_b32_e32 v7, s15
-; VI-MOVREL-NEXT: v_mov_b32_e32 v8, s16
-; VI-MOVREL-NEXT: v_mov_b32_e32 v9, s17
-; VI-MOVREL-NEXT: v_mov_b32_e32 v10, s7
-; VI-MOVREL-NEXT: v_mov_b32_e32 v11, s6
-; VI-MOVREL-NEXT: v_mov_b32_e32 v12, s5
-; VI-MOVREL-NEXT: v_mov_b32_e32 v13, s4
-; VI-MOVREL-NEXT: v_mov_b32_e32 v14, s3
-; VI-MOVREL-NEXT: v_mov_b32_e32 v15, s2
-; VI-MOVREL-NEXT: v_movrels_b32_e32 v2, v0
+; VI-MOVREL-NEXT: s_add_i32 m0, s18, 1
+; VI-MOVREL-NEXT: s_or_b32 s0, s0, 1
+; VI-MOVREL-NEXT: s_or_b32 s15, s15, 16
+; VI-MOVREL-NEXT: s_or_b32 s14, s14, 15
+; VI-MOVREL-NEXT: s_or_b32 s13, s13, 14
+; VI-MOVREL-NEXT: s_or_b32 s12, s12, 13
+; VI-MOVREL-NEXT: s_or_b32 s11, s11, 12
+; VI-MOVREL-NEXT: s_or_b32 s10, s10, 11
+; VI-MOVREL-NEXT: s_or_b32 s9, s9, 10
+; VI-MOVREL-NEXT: s_or_b32 s8, s8, 9
+; VI-MOVREL-NEXT: s_or_b32 s7, s7, 8
+; VI-MOVREL-NEXT: s_or_b32 s6, s6, 7
+; VI-MOVREL-NEXT: s_or_b32 s5, s5, 6
+; VI-MOVREL-NEXT: s_or_b32 s4, s4, 5
+; VI-MOVREL-NEXT: s_or_b32 s3, s3, 4
+; VI-MOVREL-NEXT: s_or_b32 s2, s2, 3
+; VI-MOVREL-NEXT: s_or_b32 s1, s1, 2
; VI-MOVREL-NEXT: v_mov_b32_e32 v0, s0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v16, s16
+; VI-MOVREL-NEXT: v_mov_b32_e32 v17, s17
; VI-MOVREL-NEXT: v_mov_b32_e32 v1, s1
-; VI-MOVREL-NEXT: flat_store_dword v[0:1], v2
+; VI-MOVREL-NEXT: v_mov_b32_e32 v2, s2
+; VI-MOVREL-NEXT: v_mov_b32_e32 v3, s3
+; VI-MOVREL-NEXT: v_mov_b32_e32 v4, s4
+; VI-MOVREL-NEXT: v_mov_b32_e32 v5, s5
+; VI-MOVREL-NEXT: v_mov_b32_e32 v6, s6
+; VI-MOVREL-NEXT: v_mov_b32_e32 v7, s7
+; VI-MOVREL-NEXT: v_mov_b32_e32 v8, s8
+; VI-MOVREL-NEXT: v_mov_b32_e32 v9, s9
+; VI-MOVREL-NEXT: v_mov_b32_e32 v10, s10
+; VI-MOVREL-NEXT: v_mov_b32_e32 v11, s11
+; VI-MOVREL-NEXT: v_mov_b32_e32 v12, s12
+; VI-MOVREL-NEXT: v_mov_b32_e32 v13, s13
+; VI-MOVREL-NEXT: v_mov_b32_e32 v14, s14
+; VI-MOVREL-NEXT: v_mov_b32_e32 v15, s15
+; VI-MOVREL-NEXT: v_movrels_b32_e32 v0, v0
+; VI-MOVREL-NEXT: flat_store_dword v[16:17], v0
; VI-MOVREL-NEXT: s_endpgm
;
; VI-IDXMODE-LABEL: extract_w_offset_salu_use_vector:
; VI-IDXMODE: ; %bb.0: ; %entry
-; VI-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-IDXMODE-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; VI-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-IDXMODE-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x24
+; VI-IDXMODE-NEXT: s_load_dwordx16 s[0:15], s[4:5], 0x64
; VI-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
-; VI-IDXMODE-NEXT: s_add_i32 s2, s2, 1
-; VI-IDXMODE-NEXT: s_or_b32 s10, s10, 3
-; VI-IDXMODE-NEXT: s_or_b32 s9, s9, 2
-; VI-IDXMODE-NEXT: s_or_b32 s8, s8, 1
-; VI-IDXMODE-NEXT: s_or_b32 s3, s23, 16
-; VI-IDXMODE-NEXT: s_or_b32 s4, s22, 15
-; VI-IDXMODE-NEXT: s_or_b32 s5, s21, 14
-; VI-IDXMODE-NEXT: s_or_b32 s6, s20, 13
-; VI-IDXMODE-NEXT: s_or_b32 s7, s19, 12
-; VI-IDXMODE-NEXT: s_or_b32 s18, s18, 11
-; VI-IDXMODE-NEXT: s_or_b32 s17, s17, 10
-; VI-IDXMODE-NEXT: s_or_b32 s16, s16, 9
-; VI-IDXMODE-NEXT: s_or_b32 s15, s15, 8
-; VI-IDXMODE-NEXT: s_or_b32 s14, s14, 7
-; VI-IDXMODE-NEXT: s_or_b32 s13, s13, 6
-; VI-IDXMODE-NEXT: s_or_b32 s12, s12, 5
-; VI-IDXMODE-NEXT: s_or_b32 s11, s11, 4
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, s8
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, s9
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, s10
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, s11
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, s12
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, s13
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, s14
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v7, s15
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v8, s16
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v9, s17
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v10, s18
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v11, s7
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v12, s6
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v13, s5
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v14, s4
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v15, s3
-; VI-IDXMODE-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, v0
-; VI-IDXMODE-NEXT: s_set_gpr_idx_off
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v16, s16
+; VI-IDXMODE-NEXT: s_or_b32 s0, s0, 1
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v17, s17
+; VI-IDXMODE-NEXT: s_add_i32 s16, s18, 1
+; VI-IDXMODE-NEXT: s_or_b32 s15, s15, 16
+; VI-IDXMODE-NEXT: s_or_b32 s14, s14, 15
+; VI-IDXMODE-NEXT: s_or_b32 s13, s13, 14
+; VI-IDXMODE-NEXT: s_or_b32 s12, s12, 13
+; VI-IDXMODE-NEXT: s_or_b32 s11, s11, 12
+; VI-IDXMODE-NEXT: s_or_b32 s10, s10, 11
+; VI-IDXMODE-NEXT: s_or_b32 s9, s9, 10
+; VI-IDXMODE-NEXT: s_or_b32 s8, s8, 9
+; VI-IDXMODE-NEXT: s_or_b32 s7, s7, 8
+; VI-IDXMODE-NEXT: s_or_b32 s6, s6, 7
+; VI-IDXMODE-NEXT: s_or_b32 s5, s5, 6
+; VI-IDXMODE-NEXT: s_or_b32 s4, s4, 5
+; VI-IDXMODE-NEXT: s_or_b32 s3, s3, 4
+; VI-IDXMODE-NEXT: s_or_b32 s2, s2, 3
+; VI-IDXMODE-NEXT: s_or_b32 s1, s1, 2
; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, s0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, s1
-; VI-IDXMODE-NEXT: flat_store_dword v[0:1], v2
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, s2
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, s3
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, s4
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, s5
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, s6
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v7, s7
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v8, s8
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v9, s9
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v10, s10
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v11, s11
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v12, s12
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v13, s13
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v14, s14
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v15, s15
+; VI-IDXMODE-NEXT: s_set_gpr_idx_on s16, gpr_idx(SRC0)
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, v0
+; VI-IDXMODE-NEXT: s_set_gpr_idx_off
+; VI-IDXMODE-NEXT: flat_store_dword v[16:17], v0
; VI-IDXMODE-NEXT: s_endpgm
;
; GFX9-IDXMODE-LABEL: extract_w_offset_salu_use_vector:
; GFX9-IDXMODE: ; %bb.0: ; %entry
-; GFX9-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-IDXMODE-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x64
-; GFX9-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v16, 0
; GFX9-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-IDXMODE-NEXT: s_add_i32 s2, s2, 1
@@ -631,8 +627,7 @@ entry:
define amdgpu_kernel void @extract_wo_offset(ptr addrspace(1) %out, i32 %in) {
; GENERIC-LABEL: extract_wo_offset:
; GENERIC: ; %bb.0: ; %entry
-; GENERIC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GENERIC-NEXT: s_load_dword s6, s[4:5], 0xb
+; GENERIC-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GENERIC-NEXT: s_mov_b32 s3, 0xf000
; GENERIC-NEXT: s_mov_b32 s2, -1
; GENERIC-NEXT: v_mov_b32_e32 v0, 0x40400000
@@ -649,6 +644,8 @@ define amdgpu_kernel void @extract_wo_offset(ptr addrspace(1) %out, i32 %in) {
; GENERIC-NEXT: v_mov_b32_e32 v11, 0x41700000
; GENERIC-NEXT: v_mov_b32_e32 v12, 0x41800000
; GENERIC-NEXT: s_waitcnt lgkmcnt(0)
+; GENERIC-NEXT: s_mov_b32 s0, s4
+; GENERIC-NEXT: s_mov_b32 s1, s5
; GENERIC-NEXT: s_cmp_eq_u32 s6, 1
; GENERIC-NEXT: s_cselect_b64 s[4:5], -1, 0
; GENERIC-NEXT: v_cndmask_b32_e64 v13, 1.0, 2.0, s[4:5]
@@ -769,15 +766,14 @@ define amdgpu_kernel void @extract_wo_offset(ptr addrspace(1) %out, i32 %in) {
;
; SI-MOVREL-LABEL: extract_wo_offset:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-MOVREL-NEXT: s_load_dword s4, s[4:5], 0xb
+; SI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
-; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
-; SI-MOVREL-NEXT: s_mov_b32 s2, -1
; SI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-MOVREL-NEXT: s_mov_b32 m0, s4
+; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
; SI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
+; SI-MOVREL-NEXT: s_mov_b32 m0, s2
+; SI-MOVREL-NEXT: s_mov_b32 s2, -1
; SI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
; SI-MOVREL-NEXT: v_mov_b32_e32 v5, 0x40c00000
@@ -797,14 +793,15 @@ define amdgpu_kernel void @extract_wo_offset(ptr addrspace(1) %out, i32 %in) {
;
; VI-MOVREL-LABEL: extract_wo_offset:
; VI-MOVREL: ; %bb.0: ; %entry
-; VI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-MOVREL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
; VI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; VI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
; VI-MOVREL-NEXT: s_mov_b32 m0, s2
+; VI-MOVREL-NEXT: v_mov_b32_e32 v16, s0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v17, s1
; VI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
; VI-MOVREL-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-MOVREL-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -817,20 +814,20 @@ define amdgpu_kernel void @extract_wo_offset(ptr addrspace(1) %out, i32 %in) {
; VI-MOVREL-NEXT: v_mov_b32_e32 v13, 0x41600000
; VI-MOVREL-NEXT: v_mov_b32_e32 v14, 0x41700000
; VI-MOVREL-NEXT: v_mov_b32_e32 v15, 0x41800000
-; VI-MOVREL-NEXT: v_movrels_b32_e32 v2, v0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v0, s0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v1, s1
-; VI-MOVREL-NEXT: flat_store_dword v[0:1], v2
+; VI-MOVREL-NEXT: v_movrels_b32_e32 v0, v0
+; VI-MOVREL-NEXT: flat_store_dword v[16:17], v0
; VI-MOVREL-NEXT: s_endpgm
;
; VI-IDXMODE-LABEL: extract_wo_offset:
; VI-IDXMODE: ; %bb.0: ; %entry
-; VI-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
+; VI-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v16, s0
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v17, s1
; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, 0x40a00000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -843,19 +840,15 @@ define amdgpu_kernel void @extract_wo_offset(ptr addrspace(1) %out, i32 %in) {
; VI-IDXMODE-NEXT: v_mov_b32_e32 v13, 0x41600000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v14, 0x41700000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v15, 0x41800000
-; VI-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
; VI-IDXMODE-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, v0
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, v0
; VI-IDXMODE-NEXT: s_set_gpr_idx_off
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, s0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, s1
-; VI-IDXMODE-NEXT: flat_store_dword v[0:1], v2
+; VI-IDXMODE-NEXT: flat_store_dword v[16:17], v0
; VI-IDXMODE-NEXT: s_endpgm
;
; GFX9-IDXMODE-LABEL: extract_wo_offset:
; GFX9-IDXMODE: ; %bb.0: ; %entry
-; GFX9-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v16, 0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
@@ -888,10 +881,9 @@ entry:
define amdgpu_kernel void @extract_neg_offset_sgpr(ptr addrspace(1) %out, i32 %offset) {
; GENERIC-LABEL: extract_neg_offset_sgpr:
; GENERIC: ; %bb.0: ; %entry
-; GENERIC-NEXT: s_load_dword s2, s[4:5], 0xb
-; GENERIC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GENERIC-NEXT: s_mov_b32 s3, 0xf000
+; GENERIC-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GENERIC-NEXT: s_waitcnt lgkmcnt(0)
+; GENERIC-NEXT: s_mov_b32 s3, 0xf000
; GENERIC-NEXT: s_addk_i32 s2, 0xfe00
; GENERIC-NEXT: s_cmp_eq_u32 s2, 1
; GENERIC-NEXT: s_cselect_b32 s4, 1, 0
@@ -1000,14 +992,15 @@ define amdgpu_kernel void @extract_neg_offset_sgpr(ptr addrspace(1) %out, i32 %o
;
; SI-MOVREL-LABEL: extract_neg_offset_sgpr:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-MOVREL-NEXT: s_load_dword s4, s[4:5], 0xb
+; SI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-MOVREL-NEXT: v_mov_b32_e32 v0, 0
-; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
-; SI-MOVREL-NEXT: s_mov_b32 s2, -1
+; SI-MOVREL-NEXT: s_mov_b32 s7, 0xf000
+; SI-MOVREL-NEXT: s_mov_b32 s6, -1
; SI-MOVREL-NEXT: v_mov_b32_e32 v1, 1
; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-MOVREL-NEXT: s_add_i32 m0, s4, 0xfffffe00
+; SI-MOVREL-NEXT: s_add_i32 m0, s2, 0xfffffe00
+; SI-MOVREL-NEXT: s_mov_b32 s4, s0
+; SI-MOVREL-NEXT: s_mov_b32 s5, s1
; SI-MOVREL-NEXT: v_mov_b32_e32 v2, 2
; SI-MOVREL-NEXT: v_mov_b32_e32 v3, 3
; SI-MOVREL-NEXT: v_mov_b32_e32 v4, 5
@@ -1023,19 +1016,20 @@ define amdgpu_kernel void @extract_neg_offset_sgpr(ptr addrspace(1) %out, i32 %o
; SI-MOVREL-NEXT: v_mov_b32_e32 v14, 15
; SI-MOVREL-NEXT: v_mov_b32_e32 v15, 16
; SI-MOVREL-NEXT: v_movrels_b32_e32 v0, v0
-; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-MOVREL-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-MOVREL-NEXT: s_endpgm
;
; VI-MOVREL-LABEL: extract_neg_offset_sgpr:
; VI-MOVREL: ; %bb.0: ; %entry
-; VI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-MOVREL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-MOVREL-NEXT: v_mov_b32_e32 v0, 0
; VI-MOVREL-NEXT: v_mov_b32_e32 v1, 1
; VI-MOVREL-NEXT: v_mov_b32_e32 v2, 2
; VI-MOVREL-NEXT: v_mov_b32_e32 v3, 3
; VI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
; VI-MOVREL-NEXT: s_add_i32 m0, s2, 0xfffffe00
+; VI-MOVREL-NEXT: v_mov_b32_e32 v16, s0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v17, s1
; VI-MOVREL-NEXT: v_mov_b32_e32 v4, 5
; VI-MOVREL-NEXT: v_mov_b32_e32 v5, 6
; VI-MOVREL-NEXT: v_mov_b32_e32 v6, 7
@@ -1048,22 +1042,21 @@ define amdgpu_kernel void @extract_neg_offset_sgpr(ptr addrspace(1) %out, i32 %o
; VI-MOVREL-NEXT: v_mov_b32_e32 v13, 14
; VI-MOVREL-NEXT: v_mov_b32_e32 v14, 15
; VI-MOVREL-NEXT: v_mov_b32_e32 v15, 16
-; VI-MOVREL-NEXT: v_movrels_b32_e32 v2, v0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v0, s0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v1, s1
-; VI-MOVREL-NEXT: flat_store_dword v[0:1], v2
+; VI-MOVREL-NEXT: v_movrels_b32_e32 v0, v0
+; VI-MOVREL-NEXT: flat_store_dword v[16:17], v0
; VI-MOVREL-NEXT: s_endpgm
;
; VI-IDXMODE-LABEL: extract_neg_offset_sgpr:
; VI-IDXMODE: ; %bb.0: ; %entry
-; VI-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, 0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, 1
; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, 2
; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, 3
; VI-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
-; VI-IDXMODE-NEXT: s_addk_i32 s2, 0xfe00
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v16, s0
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v17, s1
+; VI-IDXMODE-NEXT: s_add_i32 s0, s2, 0xfffffe00
; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, 5
; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, 6
; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, 7
@@ -1076,18 +1069,15 @@ define amdgpu_kernel void @extract_neg_offset_sgpr(ptr addrspace(1) %out, i32 %o
; VI-IDXMODE-NEXT: v_mov_b32_e32 v13, 14
; VI-IDXMODE-NEXT: v_mov_b32_e32 v14, 15
; VI-IDXMODE-NEXT: v_mov_b32_e32 v15, 16
-; VI-IDXMODE-NEXT: s_set_gpr_idx_on s2, gpr_idx(SRC0)
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, v0
+; VI-IDXMODE-NEXT: s_set_gpr_idx_on s0, gpr_idx(SRC0)
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, v0
; VI-IDXMODE-NEXT: s_set_gpr_idx_off
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, s0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, s1
-; VI-IDXMODE-NEXT: flat_store_dword v[0:1], v2
+; VI-IDXMODE-NEXT: flat_store_dword v[16:17], v0
; VI-IDXMODE-NEXT: s_endpgm
;
; GFX9-IDXMODE-LABEL: extract_neg_offset_sgpr:
; GFX9-IDXMODE: ; %bb.0: ; %entry
-; GFX9-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v0, 0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v1, 1
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, 2
@@ -1944,8 +1934,7 @@ entry:
define amdgpu_kernel void @insert_w_offset(ptr addrspace(1) %out, i32 %in) {
; GENERIC-LABEL: insert_w_offset:
; GENERIC: ; %bb.0: ; %entry
-; GENERIC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GENERIC-NEXT: s_load_dword s4, s[4:5], 0xb
+; GENERIC-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GENERIC-NEXT: s_mov_b32 s3, 0xf000
; GENERIC-NEXT: s_mov_b32 s2, -1
; GENERIC-NEXT: v_mov_b32_e32 v10, 0x41880000
@@ -1963,7 +1952,9 @@ define amdgpu_kernel void @insert_w_offset(ptr addrspace(1) %out, i32 %in) {
; GENERIC-NEXT: v_mov_b32_e32 v17, 0x41600000
; GENERIC-NEXT: v_mov_b32_e32 v18, 0x41500000
; GENERIC-NEXT: s_waitcnt lgkmcnt(0)
-; GENERIC-NEXT: s_add_i32 s4, s4, 1
+; GENERIC-NEXT: s_mov_b32 s0, s4
+; GENERIC-NEXT: s_mov_b32 s1, s5
+; GENERIC-NEXT: s_add_i32 s4, s6, 1
; GENERIC-NEXT: s_cmp_eq_u32 s4, 3
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v3, 4.0, v10, vcc
@@ -2140,14 +2131,13 @@ define amdgpu_kernel void @insert_w_offset(ptr addrspace(1) %out, i32 %in) {
;
; SI-MOVREL-LABEL: insert_w_offset:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-MOVREL-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
+; SI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
; SI-MOVREL-NEXT: s_add_i32 m0, s6, 1
-; SI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
; SI-MOVREL-NEXT: v_mov_b32_e32 v5, 0x40c00000
; SI-MOVREL-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -2163,6 +2153,8 @@ define amdgpu_kernel void @insert_w_offset(ptr addrspace(1) %out, i32 %in) {
; SI-MOVREL-NEXT: v_mov_b32_e32 v16, 0x41880000
; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
; SI-MOVREL-NEXT: s_mov_b32 s2, -1
+; SI-MOVREL-NEXT: s_mov_b32 s0, s4
+; SI-MOVREL-NEXT: s_mov_b32 s1, s5
; SI-MOVREL-NEXT: v_movreld_b32_e32 v0, v16
; SI-MOVREL-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
; SI-MOVREL-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
@@ -2172,15 +2164,14 @@ define amdgpu_kernel void @insert_w_offset(ptr addrspace(1) %out, i32 %in) {
;
; VI-MOVREL-LABEL: insert_w_offset:
; VI-MOVREL: ; %bb.0: ; %entry
-; VI-MOVREL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; VI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
; VI-MOVREL-NEXT: s_add_i32 m0, s2, 1
; VI-MOVREL-NEXT: s_add_u32 s2, s0, 48
-; VI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
; VI-MOVREL-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-MOVREL-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -2219,14 +2210,13 @@ define amdgpu_kernel void @insert_w_offset(ptr addrspace(1) %out, i32 %in) {
;
; VI-IDXMODE-LABEL: insert_w_offset:
; VI-IDXMODE: ; %bb.0: ; %entry
-; VI-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
; VI-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
; VI-IDXMODE-NEXT: s_add_i32 s2, s2, 1
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, 0x40a00000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -2268,14 +2258,13 @@ define amdgpu_kernel void @insert_w_offset(ptr addrspace(1) %out, i32 %in) {
;
; GFX9-IDXMODE-LABEL: insert_w_offset:
; GFX9-IDXMODE: ; %bb.0: ; %entry
-; GFX9-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v16, 0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
; GFX9-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-IDXMODE-NEXT: s_add_i32 s2, s2, 1
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v4, 0x40a00000
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v5, 0x40c00000
@@ -2308,8 +2297,7 @@ entry:
define amdgpu_kernel void @insert_unsigned_base_plus_offset(ptr addrspace(1) %out, i16 %in) {
; GENERIC-LABEL: insert_unsigned_base_plus_offset:
; GENERIC: ; %bb.0: ; %entry
-; GENERIC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GENERIC-NEXT: s_load_dword s4, s[4:5], 0xb
+; GENERIC-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GENERIC-NEXT: s_mov_b32 s3, 0xf000
; GENERIC-NEXT: s_mov_b32 s2, -1
; GENERIC-NEXT: v_mov_b32_e32 v10, 0x41880000
@@ -2327,7 +2315,9 @@ define amdgpu_kernel void @insert_unsigned_base_plus_offset(ptr addrspace(1) %ou
; GENERIC-NEXT: v_mov_b32_e32 v17, 0x41600000
; GENERIC-NEXT: v_mov_b32_e32 v18, 0x41500000
; GENERIC-NEXT: s_waitcnt lgkmcnt(0)
-; GENERIC-NEXT: s_and_b32 s4, s4, 0xffff
+; GENERIC-NEXT: s_mov_b32 s0, s4
+; GENERIC-NEXT: s_mov_b32 s1, s5
+; GENERIC-NEXT: s_and_b32 s4, s6, 0xffff
; GENERIC-NEXT: s_add_i32 s4, s4, 1
; GENERIC-NEXT: s_cmp_eq_u32 s4, 3
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
@@ -2512,14 +2502,13 @@ define amdgpu_kernel void @insert_unsigned_base_plus_offset(ptr addrspace(1) %ou
;
; SI-MOVREL-LABEL: insert_unsigned_base_plus_offset:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-MOVREL-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
+; SI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
; SI-MOVREL-NEXT: s_and_b32 m0, s6, 0xffff
-; SI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
; SI-MOVREL-NEXT: v_mov_b32_e32 v5, 0x40c00000
; SI-MOVREL-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -2535,6 +2524,8 @@ define amdgpu_kernel void @insert_unsigned_base_plus_offset(ptr addrspace(1) %ou
; SI-MOVREL-NEXT: v_mov_b32_e32 v16, 0x41880000
; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
; SI-MOVREL-NEXT: s_mov_b32 s2, -1
+; SI-MOVREL-NEXT: s_mov_b32 s0, s4
+; SI-MOVREL-NEXT: s_mov_b32 s1, s5
; SI-MOVREL-NEXT: v_movreld_b32_e32 v1, v16
; SI-MOVREL-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
; SI-MOVREL-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
@@ -2544,15 +2535,14 @@ define amdgpu_kernel void @insert_unsigned_base_plus_offset(ptr addrspace(1) %ou
;
; VI-MOVREL-LABEL: insert_unsigned_base_plus_offset:
; VI-MOVREL: ; %bb.0: ; %entry
-; VI-MOVREL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; VI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
; VI-MOVREL-NEXT: s_and_b32 m0, s2, 0xffff
; VI-MOVREL-NEXT: s_add_u32 s2, s0, 48
-; VI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
; VI-MOVREL-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-MOVREL-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -2591,14 +2581,13 @@ define amdgpu_kernel void @insert_unsigned_base_plus_offset(ptr addrspace(1) %ou
;
; VI-IDXMODE-LABEL: insert_unsigned_base_plus_offset:
; VI-IDXMODE: ; %bb.0: ; %entry
-; VI-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
; VI-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
; VI-IDXMODE-NEXT: s_and_b32 s2, s2, 0xffff
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, 0x40a00000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -2640,14 +2629,13 @@ define amdgpu_kernel void @insert_unsigned_base_plus_offset(ptr addrspace(1) %ou
;
; GFX9-IDXMODE-LABEL: insert_unsigned_base_plus_offset:
; GFX9-IDXMODE: ; %bb.0: ; %entry
-; GFX9-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v16, 0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
; GFX9-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-IDXMODE-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v4, 0x40a00000
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v5, 0x40c00000
@@ -2681,8 +2669,7 @@ entry:
define amdgpu_kernel void @insert_signed_base_plus_offset(ptr addrspace(1) %out, i16 %in) {
; GENERIC-LABEL: insert_signed_base_plus_offset:
; GENERIC: ; %bb.0: ; %entry
-; GENERIC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GENERIC-NEXT: s_load_dword s4, s[4:5], 0xb
+; GENERIC-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GENERIC-NEXT: s_mov_b32 s3, 0xf000
; GENERIC-NEXT: s_mov_b32 s2, -1
; GENERIC-NEXT: v_mov_b32_e32 v10, 0x41880000
@@ -2700,7 +2687,9 @@ define amdgpu_kernel void @insert_signed_base_plus_offset(ptr addrspace(1) %out,
; GENERIC-NEXT: v_mov_b32_e32 v17, 0x41600000
; GENERIC-NEXT: v_mov_b32_e32 v18, 0x41500000
; GENERIC-NEXT: s_waitcnt lgkmcnt(0)
-; GENERIC-NEXT: s_sext_i32_i16 s4, s4
+; GENERIC-NEXT: s_mov_b32 s0, s4
+; GENERIC-NEXT: s_mov_b32 s1, s5
+; GENERIC-NEXT: s_sext_i32_i16 s4, s6
; GENERIC-NEXT: s_add_i32 s4, s4, 1
; GENERIC-NEXT: s_cmp_eq_u32 s4, 3
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
@@ -2886,15 +2875,15 @@ define amdgpu_kernel void @insert_signed_base_plus_offset(ptr addrspace(1) %out,
;
; SI-MOVREL-LABEL: insert_signed_base_plus_offset:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-MOVREL-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
+; SI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-MOVREL-NEXT: s_mov_b32 s0, s4
; SI-MOVREL-NEXT: s_sext_i32_i16 s4, s6
; SI-MOVREL-NEXT: s_add_i32 m0, s4, 1
-; SI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
; SI-MOVREL-NEXT: v_mov_b32_e32 v5, 0x40c00000
; SI-MOVREL-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -2910,6 +2899,7 @@ define amdgpu_kernel void @insert_signed_base_plus_offset(ptr addrspace(1) %out,
; SI-MOVREL-NEXT: v_mov_b32_e32 v16, 0x41880000
; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
; SI-MOVREL-NEXT: s_mov_b32 s2, -1
+; SI-MOVREL-NEXT: s_mov_b32 s1, s5
; SI-MOVREL-NEXT: v_movreld_b32_e32 v0, v16
; SI-MOVREL-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
; SI-MOVREL-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
@@ -2919,16 +2909,15 @@ define amdgpu_kernel void @insert_signed_base_plus_offset(ptr addrspace(1) %out,
;
; VI-MOVREL-LABEL: insert_signed_base_plus_offset:
; VI-MOVREL: ; %bb.0: ; %entry
-; VI-MOVREL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; VI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
; VI-MOVREL-NEXT: s_sext_i32_i16 s2, s2
; VI-MOVREL-NEXT: s_add_i32 m0, s2, 1
; VI-MOVREL-NEXT: s_add_u32 s2, s0, 48
-; VI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
; VI-MOVREL-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-MOVREL-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -2967,15 +2956,14 @@ define amdgpu_kernel void @insert_signed_base_plus_offset(ptr addrspace(1) %out,
;
; VI-IDXMODE-LABEL: insert_signed_base_plus_offset:
; VI-IDXMODE: ; %bb.0: ; %entry
-; VI-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
; VI-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
; VI-IDXMODE-NEXT: s_sext_i32_i16 s2, s2
; VI-IDXMODE-NEXT: s_add_i32 s2, s2, 1
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, 0x40a00000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, 0x40c00000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -3017,15 +3005,14 @@ define amdgpu_kernel void @insert_signed_base_plus_offset(ptr addrspace(1) %out,
;
; GFX9-IDXMODE-LABEL: insert_signed_base_plus_offset:
; GFX9-IDXMODE: ; %bb.0: ; %entry
-; GFX9-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v16, 0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
+; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
; GFX9-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-IDXMODE-NEXT: s_sext_i32_i16 s2, s2
; GFX9-IDXMODE-NEXT: s_add_i32 s2, s2, 1
-; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v4, 0x40a00000
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v5, 0x40c00000
@@ -3061,8 +3048,7 @@ entry:
define amdgpu_kernel void @insert_wo_offset(ptr addrspace(1) %out, i32 %in) {
; GENERIC-LABEL: insert_wo_offset:
; GENERIC: ; %bb.0: ; %entry
-; GENERIC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GENERIC-NEXT: s_load_dword s4, s[4:5], 0xb
+; GENERIC-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GENERIC-NEXT: s_mov_b32 s3, 0xf000
; GENERIC-NEXT: s_mov_b32 s2, -1
; GENERIC-NEXT: v_mov_b32_e32 v10, 0x41880000
@@ -3080,56 +3066,58 @@ define amdgpu_kernel void @insert_wo_offset(ptr addrspace(1) %out, i32 %in) {
; GENERIC-NEXT: v_mov_b32_e32 v17, 0x41600000
; GENERIC-NEXT: v_mov_b32_e32 v18, 0x41500000
; GENERIC-NEXT: s_waitcnt lgkmcnt(0)
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 3
+; GENERIC-NEXT: s_mov_b32 s0, s4
+; GENERIC-NEXT: s_mov_b32 s1, s5
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 3
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v3, 4.0, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 2
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 2
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v2, v0, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 1
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 1
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v1, 2.0, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 0
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 0
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v0, 1.0, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 7
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 7
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v7, v4, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 6
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 6
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v6, v5, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 5
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 5
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v5, v8, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 4
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 4
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v4, v9, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 11
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 11
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v8, v11, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 10
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 10
; GENERIC-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: s_waitcnt expcnt(0)
; GENERIC-NEXT: v_cndmask_b32_e32 v7, v12, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 9
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 9
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v6, v13, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 8
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 8
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v5, v14, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 15
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 15
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v9, v15, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 14
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 14
; GENERIC-NEXT: buffer_store_dwordx4 v[5:8], off, s[0:3], 0 offset:32
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: s_waitcnt expcnt(0)
; GENERIC-NEXT: v_cndmask_b32_e32 v8, v16, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 13
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 13
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v7, v17, v10, vcc
-; GENERIC-NEXT: s_cmp_eq_u32 s4, 12
+; GENERIC-NEXT: s_cmp_eq_u32 s6, 12
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
; GENERIC-NEXT: v_cndmask_b32_e32 v6, v18, v10, vcc
; GENERIC-NEXT: buffer_store_dwordx4 v[6:9], off, s[0:3], 0 offset:48
@@ -3254,12 +3242,13 @@ define amdgpu_kernel void @insert_wo_offset(ptr addrspace(1) %out, i32 %in) {
;
; SI-MOVREL-LABEL: insert_wo_offset:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-MOVREL-NEXT: s_load_dword s4, s[4:5], 0xb
+; SI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
; SI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
+; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
+; SI-MOVREL-NEXT: s_mov_b32 m0, s2
; SI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
; SI-MOVREL-NEXT: v_mov_b32_e32 v5, 0x40c00000
; SI-MOVREL-NEXT: v_mov_b32_e32 v6, 0x40e00000
@@ -3273,8 +3262,6 @@ define amdgpu_kernel void @insert_wo_offset(ptr addrspace(1) %out, i32 %in) {
; SI-MOVREL-NEXT: v_mov_b32_e32 v14, 0x41700000
; SI-MOVREL-NEXT: v_mov_b32_e32 v15, 0x41800000
; SI-MOVREL-NEXT: v_mov_b32_e32 v16, 0x41880000
-; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-MOVREL-NEXT: s_mov_b32 m0, s4
; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
; SI-MOVREL-NEXT: s_mov_b32 s2, -1
; SI-MOVREL-NEXT: v_movreld_b32_e32 v0, v16
@@ -3286,8 +3273,7 @@ define amdgpu_kernel void @insert_wo_offset(ptr addrspace(1) %out, i32 %in) {
;
; VI-MOVREL-LABEL: insert_wo_offset:
; VI-MOVREL: ; %bb.0: ; %entry
-; VI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-MOVREL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; VI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
@@ -3333,8 +3319,7 @@ define amdgpu_kernel void @insert_wo_offset(ptr addrspace(1) %out, i32 %in) {
;
; VI-IDXMODE-LABEL: insert_wo_offset:
; VI-IDXMODE: ; %bb.0: ; %entry
-; VI-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
@@ -3381,8 +3366,7 @@ define amdgpu_kernel void @insert_wo_offset(ptr addrspace(1) %out, i32 %in) {
;
; GFX9-IDXMODE-LABEL: insert_wo_offset:
; GFX9-IDXMODE: ; %bb.0: ; %entry
-; GFX9-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v16, 0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
@@ -3419,64 +3403,65 @@ entry:
define amdgpu_kernel void @insert_neg_offset_sgpr(ptr addrspace(1) %in, ptr addrspace(1) %out, i32 %offset) {
; GENERIC-LABEL: insert_neg_offset_sgpr:
; GENERIC: ; %bb.0: ; %entry
-; GENERIC-NEXT: s_load_dword s6, s[4:5], 0xd
-; GENERIC-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb
+; GENERIC-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0xb
; GENERIC-NEXT: s_mov_b32 s3, 0xf000
; GENERIC-NEXT: s_mov_b32 s2, -1
; GENERIC-NEXT: s_waitcnt lgkmcnt(0)
-; GENERIC-NEXT: s_addk_i32 s6, 0xfe00
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 3
-; GENERIC-NEXT: s_cselect_b32 s4, 16, 3
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 2
-; GENERIC-NEXT: s_cselect_b32 s5, 16, 2
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 1
-; GENERIC-NEXT: v_mov_b32_e32 v3, s4
-; GENERIC-NEXT: s_cselect_b32 s4, 16, 1
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 0
-; GENERIC-NEXT: v_mov_b32_e32 v2, s5
-; GENERIC-NEXT: s_cselect_b32 s5, 16, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 7
-; GENERIC-NEXT: v_mov_b32_e32 v1, s4
-; GENERIC-NEXT: s_cselect_b32 s4, 16, 7
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 6
-; GENERIC-NEXT: v_mov_b32_e32 v0, s5
-; GENERIC-NEXT: s_cselect_b32 s5, 16, 6
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 5
-; GENERIC-NEXT: v_mov_b32_e32 v7, s4
-; GENERIC-NEXT: s_cselect_b32 s4, 16, 5
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 4
-; GENERIC-NEXT: v_mov_b32_e32 v6, s5
-; GENERIC-NEXT: s_cselect_b32 s5, 16, 4
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 11
-; GENERIC-NEXT: v_mov_b32_e32 v5, s4
-; GENERIC-NEXT: s_cselect_b32 s4, 16, 11
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 10
-; GENERIC-NEXT: v_mov_b32_e32 v4, s5
+; GENERIC-NEXT: s_mov_b32 s0, s4
+; GENERIC-NEXT: s_mov_b32 s1, s5
+; GENERIC-NEXT: s_add_i32 s4, s6, 0xfffffe00
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 3
+; GENERIC-NEXT: s_cselect_b32 s5, 16, 3
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 2
+; GENERIC-NEXT: s_cselect_b32 s6, 16, 2
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 1
+; GENERIC-NEXT: v_mov_b32_e32 v3, s5
+; GENERIC-NEXT: s_cselect_b32 s5, 16, 1
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 0
+; GENERIC-NEXT: v_mov_b32_e32 v2, s6
+; GENERIC-NEXT: s_cselect_b32 s6, 16, 0
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 7
+; GENERIC-NEXT: v_mov_b32_e32 v1, s5
+; GENERIC-NEXT: s_cselect_b32 s5, 16, 7
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 6
+; GENERIC-NEXT: v_mov_b32_e32 v0, s6
+; GENERIC-NEXT: s_cselect_b32 s6, 16, 6
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 5
+; GENERIC-NEXT: v_mov_b32_e32 v7, s5
+; GENERIC-NEXT: s_cselect_b32 s5, 16, 5
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 4
+; GENERIC-NEXT: v_mov_b32_e32 v6, s6
+; GENERIC-NEXT: s_cselect_b32 s6, 16, 4
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 11
+; GENERIC-NEXT: v_mov_b32_e32 v5, s5
+; GENERIC-NEXT: s_cselect_b32 s5, 16, 11
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 10
+; GENERIC-NEXT: v_mov_b32_e32 v4, s6
; GENERIC-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
-; GENERIC-NEXT: s_cselect_b32 s5, 16, 10
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 9
+; GENERIC-NEXT: s_cselect_b32 s6, 16, 10
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 9
; GENERIC-NEXT: s_waitcnt expcnt(0)
-; GENERIC-NEXT: v_mov_b32_e32 v7, s4
-; GENERIC-NEXT: s_cselect_b32 s4, 16, 9
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 8
-; GENERIC-NEXT: v_mov_b32_e32 v6, s5
-; GENERIC-NEXT: s_cselect_b32 s5, 16, 8
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 15
-; GENERIC-NEXT: v_mov_b32_e32 v5, s4
-; GENERIC-NEXT: s_cselect_b32 s4, 16, 15
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 14
-; GENERIC-NEXT: v_mov_b32_e32 v4, s5
-; GENERIC-NEXT: s_cselect_b32 s5, 16, 14
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 13
+; GENERIC-NEXT: v_mov_b32_e32 v7, s5
+; GENERIC-NEXT: s_cselect_b32 s5, 16, 9
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 8
+; GENERIC-NEXT: v_mov_b32_e32 v6, s6
+; GENERIC-NEXT: s_cselect_b32 s6, 16, 8
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 15
+; GENERIC-NEXT: v_mov_b32_e32 v5, s5
+; GENERIC-NEXT: s_cselect_b32 s5, 16, 15
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 14
+; GENERIC-NEXT: v_mov_b32_e32 v4, s6
+; GENERIC-NEXT: s_cselect_b32 s6, 16, 14
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 13
; GENERIC-NEXT: s_cselect_b32 s7, 16, 13
-; GENERIC-NEXT: s_cmp_eq_u32 s6, 12
-; GENERIC-NEXT: s_cselect_b32 s6, 16, 12
+; GENERIC-NEXT: s_cmp_eq_u32 s4, 12
+; GENERIC-NEXT: s_cselect_b32 s4, 16, 12
; GENERIC-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:32
; GENERIC-NEXT: s_waitcnt expcnt(0)
-; GENERIC-NEXT: v_mov_b32_e32 v7, s4
-; GENERIC-NEXT: v_mov_b32_e32 v6, s5
+; GENERIC-NEXT: v_mov_b32_e32 v7, s5
+; GENERIC-NEXT: v_mov_b32_e32 v6, s6
; GENERIC-NEXT: v_mov_b32_e32 v5, s7
-; GENERIC-NEXT: v_mov_b32_e32 v4, s6
+; GENERIC-NEXT: v_mov_b32_e32 v4, s4
; GENERIC-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:48
; GENERIC-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GENERIC-NEXT: s_endpgm
@@ -3598,8 +3583,7 @@ define amdgpu_kernel void @insert_neg_offset_sgpr(ptr addrspace(1) %in, ptr addr
;
; SI-MOVREL-LABEL: insert_neg_offset_sgpr:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb
-; SI-MOVREL-NEXT: s_load_dword s4, s[4:5], 0xd
+; SI-MOVREL-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0xb
; SI-MOVREL-NEXT: v_mov_b32_e32 v0, 0
; SI-MOVREL-NEXT: v_mov_b32_e32 v1, 1
; SI-MOVREL-NEXT: v_mov_b32_e32 v2, 2
@@ -3617,9 +3601,11 @@ define amdgpu_kernel void @insert_neg_offset_sgpr(ptr addrspace(1) %in, ptr addr
; SI-MOVREL-NEXT: v_mov_b32_e32 v14, 14
; SI-MOVREL-NEXT: v_mov_b32_e32 v15, 15
; SI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-MOVREL-NEXT: s_add_i32 m0, s4, 0xfffffe00
+; SI-MOVREL-NEXT: s_add_i32 m0, s6, 0xfffffe00
; SI-MOVREL-NEXT: s_mov_b32 s3, 0xf000
; SI-MOVREL-NEXT: s_mov_b32 s2, -1
+; SI-MOVREL-NEXT: s_mov_b32 s0, s4
+; SI-MOVREL-NEXT: s_mov_b32 s1, s5
; SI-MOVREL-NEXT: v_movreld_b32_e32 v0, 16
; SI-MOVREL-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
; SI-MOVREL-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
@@ -3629,8 +3615,7 @@ define amdgpu_kernel void @insert_neg_offset_sgpr(ptr addrspace(1) %in, ptr addr
;
; VI-MOVREL-LABEL: insert_neg_offset_sgpr:
; VI-MOVREL: ; %bb.0: ; %entry
-; VI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
-; VI-MOVREL-NEXT: s_load_dword s2, s[4:5], 0x34
+; VI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; VI-MOVREL-NEXT: v_mov_b32_e32 v0, 0
; VI-MOVREL-NEXT: v_mov_b32_e32 v1, 1
; VI-MOVREL-NEXT: v_mov_b32_e32 v2, 2
@@ -3675,8 +3660,7 @@ define amdgpu_kernel void @insert_neg_offset_sgpr(ptr addrspace(1) %in, ptr addr
;
; VI-IDXMODE-LABEL: insert_neg_offset_sgpr:
; VI-IDXMODE: ; %bb.0: ; %entry
-; VI-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
-; VI-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x34
+; VI-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, 0
; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, 1
; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, 2
@@ -3723,8 +3707,7 @@ define amdgpu_kernel void @insert_neg_offset_sgpr(ptr addrspace(1) %in, ptr addr
;
; GFX9-IDXMODE-LABEL: insert_neg_offset_sgpr:
; GFX9-IDXMODE: ; %bb.0: ; %entry
-; GFX9-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
-; GFX9-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x34
+; GFX9-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v1, 1
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, 2
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v3, 3
@@ -6763,8 +6746,7 @@ bb2:
define amdgpu_kernel void @insert_w_offset_multiple_in_block(ptr addrspace(1) %out1, i32 %in) {
; GENERIC-LABEL: insert_w_offset_multiple_in_block:
; GENERIC: ; %bb.0: ; %entry
-; GENERIC-NEXT: s_load_dwordx2 s[28:29], s[4:5], 0x9
-; GENERIC-NEXT: s_load_dword s24, s[4:5], 0xb
+; GENERIC-NEXT: s_load_dwordx4 s[24:27], s[4:5], 0x9
; GENERIC-NEXT: s_mov_b32 s31, 0xf000
; GENERIC-NEXT: s_mov_b32 s30, -1
; GENERIC-NEXT: v_mov_b32_e32 v1, 0x41500000
@@ -6782,44 +6764,46 @@ define amdgpu_kernel void @insert_w_offset_multiple_in_block(ptr addrspace(1) %o
; GENERIC-NEXT: v_mov_b32_e32 v12, 0x41000000
; GENERIC-NEXT: v_mov_b32_e32 v15, 0x40400000
; GENERIC-NEXT: s_waitcnt lgkmcnt(0)
-; GENERIC-NEXT: s_add_i32 s25, s24, 1
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 12
+; GENERIC-NEXT: s_mov_b32 s28, s24
+; GENERIC-NEXT: s_mov_b32 s29, s25
+; GENERIC-NEXT: s_add_i32 s24, s26, 1
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 12
; GENERIC-NEXT: s_cselect_b64 vcc, -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 13
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 13
; GENERIC-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 14
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 14
; GENERIC-NEXT: s_cselect_b64 s[2:3], -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 15
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 15
; GENERIC-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 8
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 8
; GENERIC-NEXT: s_cselect_b64 s[6:7], -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 9
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 9
; GENERIC-NEXT: s_cselect_b64 s[8:9], -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 10
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 10
; GENERIC-NEXT: s_cselect_b64 s[10:11], -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 11
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 11
; GENERIC-NEXT: s_cselect_b64 s[12:13], -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 4
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 4
; GENERIC-NEXT: s_cselect_b64 s[14:15], -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 5
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 5
; GENERIC-NEXT: s_cselect_b64 s[16:17], -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 6
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 6
; GENERIC-NEXT: s_cselect_b64 s[18:19], -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 7
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 7
; GENERIC-NEXT: s_cselect_b64 s[20:21], -1, 0
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 0
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 0
; GENERIC-NEXT: s_cselect_b64 s[22:23], -1, 0
; GENERIC-NEXT: v_cndmask_b32_e64 v13, 1.0, v0, s[22:23]
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 1
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 1
; GENERIC-NEXT: s_cselect_b64 s[22:23], -1, 0
; GENERIC-NEXT: v_cndmask_b32_e64 v14, 2.0, v0, s[22:23]
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 2
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 2
; GENERIC-NEXT: s_cselect_b64 s[22:23], -1, 0
; GENERIC-NEXT: v_cndmask_b32_e64 v15, v15, v0, s[22:23]
-; GENERIC-NEXT: s_cmp_eq_u32 s25, 3
+; GENERIC-NEXT: s_cmp_eq_u32 s24, 3
; GENERIC-NEXT: s_cselect_b64 s[22:23], -1, 0
; GENERIC-NEXT: v_cndmask_b32_e64 v16, 4.0, v0, s[22:23]
-; GENERIC-NEXT: s_add_i32 s26, s24, 2
+; GENERIC-NEXT: s_add_i32 s26, s26, 2
; GENERIC-NEXT: s_cmp_lg_u32 s26, 3
; GENERIC-NEXT: buffer_store_dwordx4 v[13:16], off, s[28:31], 0
; GENERIC-NEXT: s_cselect_b64 s[22:23], -1, 0
@@ -7087,8 +7071,7 @@ define amdgpu_kernel void @insert_w_offset_multiple_in_block(ptr addrspace(1) %o
;
; SI-MOVREL-LABEL: insert_w_offset_multiple_in_block:
; SI-MOVREL: ; %bb.0: ; %entry
-; SI-MOVREL-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
; SI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
@@ -7141,133 +7124,134 @@ define amdgpu_kernel void @insert_w_offset_multiple_in_block(ptr addrspace(1) %o
;
; VI-MOVREL-LABEL: insert_w_offset_multiple_in_block:
; VI-MOVREL: ; %bb.0: ; %entry
-; VI-MOVREL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-MOVREL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-MOVREL-NEXT: v_mov_b32_e32 v0, 1.0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v1, 2.0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v2, 0x40400000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v3, 4.0
+; VI-MOVREL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-MOVREL-NEXT: v_mov_b32_e32 v16, 1.0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v17, 2.0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v18, 0x40400000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v19, 4.0
; VI-MOVREL-NEXT: s_waitcnt lgkmcnt(0)
; VI-MOVREL-NEXT: s_add_i32 m0, s2, 1
-; VI-MOVREL-NEXT: v_mov_b32_e32 v4, 0x40a00000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v5, 0x40c00000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v6, 0x40e00000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v7, 0x41000000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v8, 0x41100000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v9, 0x41200000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v10, 0x41300000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v11, 0x41400000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v12, 0x41500000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v13, 0x41600000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v14, 0x41700000
-; VI-MOVREL-NEXT: v_mov_b32_e32 v15, 0x41800000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v20, 0x40a00000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v21, 0x40c00000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v22, 0x40e00000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v23, 0x41000000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v24, 0x41100000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v25, 0x41200000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v26, 0x41300000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v27, 0x41400000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v28, 0x41500000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v29, 0x41600000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v30, 0x41700000
+; VI-MOVREL-NEXT: v_mov_b32_e32 v31, 0x41800000
; VI-MOVREL-NEXT: v_mov_b32_e32 v32, 0x41880000
-; VI-MOVREL-NEXT: v_movreld_b32_e32 v0, v32
+; VI-MOVREL-NEXT: v_movreld_b32_e32 v16, v32
; VI-MOVREL-NEXT: s_add_i32 m0, s2, 2
-; VI-MOVREL-NEXT: v_mov_b32_e32 v31, v15
+; VI-MOVREL-NEXT: v_mov_b32_e32 v0, v16
; VI-MOVREL-NEXT: s_add_u32 s2, s0, 48
-; VI-MOVREL-NEXT: v_mov_b32_e32 v30, v14
-; VI-MOVREL-NEXT: v_mov_b32_e32 v29, v13
-; VI-MOVREL-NEXT: v_mov_b32_e32 v28, v12
-; VI-MOVREL-NEXT: v_mov_b32_e32 v27, v11
-; VI-MOVREL-NEXT: v_mov_b32_e32 v26, v10
-; VI-MOVREL-NEXT: v_mov_b32_e32 v25, v9
-; VI-MOVREL-NEXT: v_mov_b32_e32 v24, v8
-; VI-MOVREL-NEXT: v_mov_b32_e32 v23, v7
-; VI-MOVREL-NEXT: v_mov_b32_e32 v22, v6
-; VI-MOVREL-NEXT: v_mov_b32_e32 v21, v5
-; VI-MOVREL-NEXT: v_mov_b32_e32 v20, v4
-; VI-MOVREL-NEXT: v_mov_b32_e32 v19, v3
-; VI-MOVREL-NEXT: v_mov_b32_e32 v18, v2
-; VI-MOVREL-NEXT: v_mov_b32_e32 v17, v1
-; VI-MOVREL-NEXT: v_mov_b32_e32 v16, v0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v1, v17
+; VI-MOVREL-NEXT: v_mov_b32_e32 v2, v18
+; VI-MOVREL-NEXT: v_mov_b32_e32 v3, v19
+; VI-MOVREL-NEXT: v_mov_b32_e32 v4, v20
+; VI-MOVREL-NEXT: v_mov_b32_e32 v5, v21
+; VI-MOVREL-NEXT: v_mov_b32_e32 v6, v22
+; VI-MOVREL-NEXT: v_mov_b32_e32 v7, v23
+; VI-MOVREL-NEXT: v_mov_b32_e32 v8, v24
+; VI-MOVREL-NEXT: v_mov_b32_e32 v9, v25
+; VI-MOVREL-NEXT: v_mov_b32_e32 v10, v26
+; VI-MOVREL-NEXT: v_mov_b32_e32 v11, v27
+; VI-MOVREL-NEXT: v_mov_b32_e32 v12, v28
+; VI-MOVREL-NEXT: v_mov_b32_e32 v13, v29
+; VI-MOVREL-NEXT: v_mov_b32_e32 v14, v30
+; VI-MOVREL-NEXT: v_mov_b32_e32 v15, v31
; VI-MOVREL-NEXT: s_addc_u32 s3, s1, 0
-; VI-MOVREL-NEXT: v_movreld_b32_e32 v16, v32
+; VI-MOVREL-NEXT: v_movreld_b32_e32 v0, v32
; VI-MOVREL-NEXT: v_mov_b32_e32 v33, s3
; VI-MOVREL-NEXT: v_mov_b32_e32 v32, s2
; VI-MOVREL-NEXT: s_add_u32 s2, s0, 32
; VI-MOVREL-NEXT: s_addc_u32 s3, s1, 0
-; VI-MOVREL-NEXT: flat_store_dwordx4 v[32:33], v[12:15]
+; VI-MOVREL-NEXT: flat_store_dwordx4 v[32:33], v[28:31]
; VI-MOVREL-NEXT: s_nop 0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v13, s3
-; VI-MOVREL-NEXT: v_mov_b32_e32 v12, s2
+; VI-MOVREL-NEXT: v_mov_b32_e32 v29, s3
+; VI-MOVREL-NEXT: v_mov_b32_e32 v28, s2
; VI-MOVREL-NEXT: s_add_u32 s2, s0, 16
; VI-MOVREL-NEXT: s_addc_u32 s3, s1, 0
-; VI-MOVREL-NEXT: flat_store_dwordx4 v[12:13], v[8:11]
+; VI-MOVREL-NEXT: flat_store_dwordx4 v[28:29], v[24:27]
; VI-MOVREL-NEXT: s_nop 0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v9, s3
-; VI-MOVREL-NEXT: v_mov_b32_e32 v8, s2
+; VI-MOVREL-NEXT: v_mov_b32_e32 v25, s3
+; VI-MOVREL-NEXT: v_mov_b32_e32 v24, s2
; VI-MOVREL-NEXT: s_add_u32 s2, s0, 64
-; VI-MOVREL-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; VI-MOVREL-NEXT: s_addc_u32 s3, s1, 0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v5, s1
-; VI-MOVREL-NEXT: v_mov_b32_e32 v4, s0
+; VI-MOVREL-NEXT: flat_store_dwordx4 v[24:25], v[20:23]
; VI-MOVREL-NEXT: s_add_u32 s4, s0, 0x70
-; VI-MOVREL-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-MOVREL-NEXT: v_mov_b32_e32 v21, s1
+; VI-MOVREL-NEXT: v_mov_b32_e32 v20, s0
; VI-MOVREL-NEXT: s_addc_u32 s5, s1, 0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v0, s4
-; VI-MOVREL-NEXT: v_mov_b32_e32 v1, s5
+; VI-MOVREL-NEXT: flat_store_dwordx4 v[20:21], v[16:19]
+; VI-MOVREL-NEXT: s_nop 0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v17, s5
+; VI-MOVREL-NEXT: v_mov_b32_e32 v16, s4
; VI-MOVREL-NEXT: s_add_u32 s4, s0, 0x60
-; VI-MOVREL-NEXT: flat_store_dwordx4 v[0:1], v[28:31]
; VI-MOVREL-NEXT: s_addc_u32 s5, s1, 0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v0, s4
-; VI-MOVREL-NEXT: v_mov_b32_e32 v1, s5
+; VI-MOVREL-NEXT: flat_store_dwordx4 v[16:17], v[12:15]
; VI-MOVREL-NEXT: s_add_u32 s0, s0, 0x50
-; VI-MOVREL-NEXT: flat_store_dwordx4 v[0:1], v[24:27]
+; VI-MOVREL-NEXT: v_mov_b32_e32 v13, s5
+; VI-MOVREL-NEXT: v_mov_b32_e32 v12, s4
; VI-MOVREL-NEXT: s_addc_u32 s1, s1, 0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v0, s0
-; VI-MOVREL-NEXT: v_mov_b32_e32 v1, s1
-; VI-MOVREL-NEXT: flat_store_dwordx4 v[0:1], v[20:23]
-; VI-MOVREL-NEXT: v_mov_b32_e32 v0, s2
-; VI-MOVREL-NEXT: v_mov_b32_e32 v1, s3
-; VI-MOVREL-NEXT: flat_store_dwordx4 v[0:1], v[16:19]
+; VI-MOVREL-NEXT: flat_store_dwordx4 v[12:13], v[8:11]
+; VI-MOVREL-NEXT: s_nop 0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v9, s1
+; VI-MOVREL-NEXT: v_mov_b32_e32 v8, s0
+; VI-MOVREL-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
+; VI-MOVREL-NEXT: s_nop 0
+; VI-MOVREL-NEXT: v_mov_b32_e32 v5, s3
+; VI-MOVREL-NEXT: v_mov_b32_e32 v4, s2
+; VI-MOVREL-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-MOVREL-NEXT: s_endpgm
;
; VI-IDXMODE-LABEL: insert_w_offset_multiple_in_block:
; VI-IDXMODE: ; %bb.0: ; %entry
-; VI-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, 4.0
+; VI-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v16, 1.0
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v17, 2.0
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v18, 0x40400000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v19, 4.0
; VI-IDXMODE-NEXT: s_waitcnt lgkmcnt(0)
; VI-IDXMODE-NEXT: s_add_i32 s3, s2, 1
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, 0x40a00000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, 0x40c00000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, 0x40e00000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v7, 0x41000000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v8, 0x41100000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v9, 0x41200000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v10, 0x41300000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v11, 0x41400000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v12, 0x41500000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v13, 0x41600000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v14, 0x41700000
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v15, 0x41800000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v20, 0x40a00000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v21, 0x40c00000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v22, 0x40e00000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v23, 0x41000000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v24, 0x41100000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v25, 0x41200000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v26, 0x41300000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v27, 0x41400000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v28, 0x41500000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v29, 0x41600000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v30, 0x41700000
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v31, 0x41800000
; VI-IDXMODE-NEXT: v_mov_b32_e32 v32, 0x41880000
; VI-IDXMODE-NEXT: s_set_gpr_idx_on s3, gpr_idx(DST)
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, v32
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v16, v32
; VI-IDXMODE-NEXT: s_set_gpr_idx_off
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v31, v15
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, v16
; VI-IDXMODE-NEXT: s_add_i32 s2, s2, 2
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v30, v14
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v29, v13
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v28, v12
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v27, v11
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v26, v10
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v25, v9
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v24, v8
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v23, v7
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v22, v6
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v21, v5
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v20, v4
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v19, v3
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v18, v2
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v17, v1
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v16, v0
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, v17
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v2, v18
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v3, v19
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, v20
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, v21
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v6, v22
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v7, v23
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v8, v24
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v9, v25
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v10, v26
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v11, v27
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v12, v28
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v13, v29
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v14, v30
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v15, v31
; VI-IDXMODE-NEXT: s_set_gpr_idx_on s2, gpr_idx(DST)
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v16, v32
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, v32
; VI-IDXMODE-NEXT: s_set_gpr_idx_off
; VI-IDXMODE-NEXT: s_add_u32 s2, s0, 48
; VI-IDXMODE-NEXT: s_addc_u32 s3, s1, 0
@@ -7275,46 +7259,48 @@ define amdgpu_kernel void @insert_w_offset_multiple_in_block(ptr addrspace(1) %o
; VI-IDXMODE-NEXT: v_mov_b32_e32 v32, s2
; VI-IDXMODE-NEXT: s_add_u32 s2, s0, 32
; VI-IDXMODE-NEXT: s_addc_u32 s3, s1, 0
-; VI-IDXMODE-NEXT: flat_store_dwordx4 v[32:33], v[12:15]
+; VI-IDXMODE-NEXT: flat_store_dwordx4 v[32:33], v[28:31]
; VI-IDXMODE-NEXT: s_nop 0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v13, s3
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v12, s2
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v29, s3
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v28, s2
; VI-IDXMODE-NEXT: s_add_u32 s2, s0, 16
; VI-IDXMODE-NEXT: s_addc_u32 s3, s1, 0
-; VI-IDXMODE-NEXT: flat_store_dwordx4 v[12:13], v[8:11]
+; VI-IDXMODE-NEXT: flat_store_dwordx4 v[28:29], v[24:27]
; VI-IDXMODE-NEXT: s_nop 0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v9, s3
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v8, s2
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v25, s3
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v24, s2
; VI-IDXMODE-NEXT: s_add_u32 s2, s0, 64
-; VI-IDXMODE-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
; VI-IDXMODE-NEXT: s_addc_u32 s3, s1, 0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, s1
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, s0
+; VI-IDXMODE-NEXT: flat_store_dwordx4 v[24:25], v[20:23]
; VI-IDXMODE-NEXT: s_add_u32 s4, s0, 0x70
-; VI-IDXMODE-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v21, s1
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v20, s0
; VI-IDXMODE-NEXT: s_addc_u32 s5, s1, 0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, s4
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, s5
+; VI-IDXMODE-NEXT: flat_store_dwordx4 v[20:21], v[16:19]
+; VI-IDXMODE-NEXT: s_nop 0
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v17, s5
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v16, s4
; VI-IDXMODE-NEXT: s_add_u32 s4, s0, 0x60
-; VI-IDXMODE-NEXT: flat_store_dwordx4 v[0:1], v[28:31]
; VI-IDXMODE-NEXT: s_addc_u32 s5, s1, 0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, s4
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, s5
+; VI-IDXMODE-NEXT: flat_store_dwordx4 v[16:17], v[12:15]
; VI-IDXMODE-NEXT: s_add_u32 s0, s0, 0x50
-; VI-IDXMODE-NEXT: flat_store_dwordx4 v[0:1], v[24:27]
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v13, s5
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v12, s4
; VI-IDXMODE-NEXT: s_addc_u32 s1, s1, 0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, s0
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, s1
-; VI-IDXMODE-NEXT: flat_store_dwordx4 v[0:1], v[20:23]
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v0, s2
-; VI-IDXMODE-NEXT: v_mov_b32_e32 v1, s3
-; VI-IDXMODE-NEXT: flat_store_dwordx4 v[0:1], v[16:19]
+; VI-IDXMODE-NEXT: flat_store_dwordx4 v[12:13], v[8:11]
+; VI-IDXMODE-NEXT: s_nop 0
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v9, s1
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v8, s0
+; VI-IDXMODE-NEXT: flat_store_dwordx4 v[8:9], v[4:7]
+; VI-IDXMODE-NEXT: s_nop 0
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v5, s3
+; VI-IDXMODE-NEXT: v_mov_b32_e32 v4, s2
+; VI-IDXMODE-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; VI-IDXMODE-NEXT: s_endpgm
;
; GFX9-IDXMODE-LABEL: insert_w_offset_multiple_in_block:
; GFX9-IDXMODE: ; %bb.0: ; %entry
-; GFX9-IDXMODE-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-IDXMODE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-IDXMODE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v0, 1.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v1, 2.0
; GFX9-IDXMODE-NEXT: v_mov_b32_e32 v2, 0x40400000
diff --git a/llvm/test/CodeGen/AMDGPU/inlineasm-sgmask.ll b/llvm/test/CodeGen/AMDGPU/inlineasm-sgmask.ll
index 77320094b1b65..27e19f2416fc4 100644
--- a/llvm/test/CodeGen/AMDGPU/inlineasm-sgmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/inlineasm-sgmask.ll
@@ -44,6 +44,7 @@ define protected amdgpu_kernel void @test_salu(ptr addrspace(1) noalias noundef
; CHECK: %bb.1
; CHECK-NEXT: s_load
; CHECK-NEXT: s_load
+; CHECK: killed
; CHECK-NEXT: s_waitcnt
; CHECK-NEXT: ASMSTART
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
index 8075d5fa19b77..ebb0e179b2568 100644
--- a/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert-waitcnts-merge.ll
@@ -15,14 +15,13 @@ define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1) #0 {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $sgpr22_sgpr23 = S_MOV_B64 $sgpr2_sgpr3, implicit $sgpr0_sgpr1_sgpr2_sgpr3, implicit-def $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: $sgpr20_sgpr21 = S_MOV_B64 killed $sgpr0_sgpr1, implicit $sgpr0_sgpr1_sgpr2_sgpr3
- ; CHECK-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr8_sgpr9, 8, 0 :: (dereferenceable invariant load (s32) from %ir.arg1.kernarg.offset.align.down, align 8, addrspace 4)
- ; CHECK-NEXT: renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM killed renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s64) from %ir.arg.kernarg.offset1, align 16, addrspace 4)
+ ; CHECK-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr8_sgpr9, 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg.kernarg.offset1, addrspace 4)
; CHECK-NEXT: $sgpr20 = S_ADD_U32 $sgpr20, killed $sgpr17, implicit-def $scc, implicit-def $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: $sgpr21 = S_ADDC_U32 $sgpr21, 0, implicit-def dead $scc, implicit killed $scc, implicit-def $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: renamable $vgpr1 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: renamable $sgpr4_sgpr5 = S_MOV_B64 0
; CHECK-NEXT: S_WAITCNT .Lgkmcnt_0
; CHECK-NEXT: S_BITCMP1_B32 killed renamable $sgpr2, 0, implicit-def $scc
- ; CHECK-NEXT: renamable $sgpr4_sgpr5 = S_MOV_B64 0
; CHECK-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
; CHECK-NEXT: renamable $sgpr2_sgpr3 = S_CSELECT_B64 -1, 0, implicit killed $scc
; CHECK-NEXT: $vgpr2_vgpr3 = V_PK_MOV_B32 8, 0, 8, 0, 0, 0, 0, 0, 0, implicit $exec
@@ -31,7 +30,7 @@ define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1) #0 {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.10.loop.exit.guard:
; CHECK-NEXT: successors: %bb.11(0x04000000), %bb.1(0x7c000000)
- ; CHECK-NEXT: liveins: $vgpr0, $sgpr0_sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr2_sgpr3, $sgpr4_sgpr5, $sgpr0_sgpr1_sgpr2_sgpr3:0x000000000000000F, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: renamable $vcc = S_AND_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def dead $scc
; CHECK-NEXT: renamable $sgpr4_sgpr5 = S_MOV_B64 0
@@ -40,7 +39,7 @@ define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1) #0 {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.1.bb2:
; CHECK-NEXT: successors: %bb.12(0x40000000), %bb.2(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0, $sgpr0_sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5, $sgpr6_sgpr7, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr2_sgpr3, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x000000000000000F, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_WAITCNT .Vmcnt_0
; CHECK-NEXT: renamable $vgpr0 = V_XOR_B32_e32 1, killed $vgpr0, implicit $exec
@@ -55,14 +54,14 @@ define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1) #0 {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.12.bb13:
; CHECK-NEXT: successors: %bb.2(0x80000000)
- ; CHECK-NEXT: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
+ ; CHECK-NEXT: liveins: $sgpr2_sgpr3, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr0_sgpr1_sgpr2_sgpr3:0x000000000000000F, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: renamable $vgpr0 = GLOBAL_LOAD_DWORD renamable $vgpr2_vgpr3, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(1) null`, addrspace 1)
; CHECK-NEXT: renamable $sgpr6_sgpr7 = S_ANDN2_B64 killed renamable $sgpr6_sgpr7, $exec, implicit-def dead $scc
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.2.Flow3:
+ ; CHECK-NEXT: bb.2.Flow4:
; CHECK-NEXT: successors: %bb.3(0x40000000), %bb.1(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0, $sgpr0_sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr2_sgpr3, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr8_sgpr9, $sgpr0_sgpr1_sgpr2_sgpr3:0x000000000000000F, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr8_sgpr9, implicit-def $scc
; CHECK-NEXT: renamable $sgpr8_sgpr9 = S_AND_B64 $exec, renamable $sgpr6_sgpr7, implicit-def $scc
@@ -72,23 +71,23 @@ define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1) #0 {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.3.bb3:
; CHECK-NEXT: successors: %bb.4(0x80000000)
- ; CHECK-NEXT: liveins: $sgpr0_sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
+ ; CHECK-NEXT: liveins: $sgpr2_sgpr3, $sgpr4_sgpr5, $sgpr0_sgpr1_sgpr2_sgpr3:0x000000000000000F, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $exec = S_OR_B64 $exec, killed renamable $sgpr4_sgpr5, implicit-def $scc
; CHECK-NEXT: renamable $vgpr0 = BUFFER_LOAD_DWORD_OFFSET $sgpr20_sgpr21_sgpr22_sgpr23, 0, 0, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(5) null`, addrspace 5)
; CHECK-NEXT: renamable $vgpr4 = GLOBAL_LOAD_DWORD renamable $vgpr2_vgpr3, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(1) null`, addrspace 1)
; CHECK-NEXT: S_BRANCH %bb.4
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: bb.9.Flow2:
+ ; CHECK-NEXT: bb.9.Flow3:
; CHECK-NEXT: successors: %bb.10(0x04000000), %bb.4(0x7c000000)
- ; CHECK-NEXT: liveins: $vgpr0, $vgpr4, $sgpr0_sgpr1, $sgpr2_sgpr3, $sgpr4_sgpr5, $sgpr6_sgpr7, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr4, $sgpr2_sgpr3, $sgpr4_sgpr5, $sgpr6_sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x000000000000000F, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr6_sgpr7, implicit-def dead $scc
; CHECK-NEXT: S_CBRANCH_VCCZ %bb.10, implicit killed $vcc
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.4.bb6:
; CHECK-NEXT: successors: %bb.6(0x40000000), %bb.5(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0, $vgpr4, $sgpr0_sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr4, $sgpr2_sgpr3, $sgpr0_sgpr1_sgpr2_sgpr3:0x000000000000000F, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: S_WAITCNT .Vmcnt_0
; CHECK-NEXT: renamable $sgpr6_sgpr7 = V_CMP_EQ_U32_e64 0, killed $vgpr4, implicit $exec
@@ -97,22 +96,22 @@ define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1) #0 {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.6.bb9:
; CHECK-NEXT: successors: %bb.7(0x80000000)
- ; CHECK-NEXT: liveins: $vgpr0, $sgpr0_sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr2_sgpr3, $sgpr0_sgpr1_sgpr2_sgpr3:0x000000000000000F, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR renamable $vgpr1, renamable $vgpr1, renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s32) into %ir.arg.load, addrspace 1)
+ ; CHECK-NEXT: GLOBAL_STORE_DWORD_SADDR renamable $vgpr1, renamable $vgpr1, renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store (s32) into %ir.1, addrspace 1)
; CHECK-NEXT: renamable $vgpr4 = GLOBAL_LOAD_DWORD renamable $vgpr2_vgpr3, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(1) null`, addrspace 1)
; CHECK-NEXT: renamable $sgpr6_sgpr7 = S_MOV_B64 -1
; CHECK-NEXT: S_BRANCH %bb.7
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.5:
; CHECK-NEXT: successors: %bb.7(0x80000000)
- ; CHECK-NEXT: liveins: $vgpr0, $sgpr0_sgpr1, $sgpr2_sgpr3, $sgpr6_sgpr7, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr2_sgpr3, $sgpr6_sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x000000000000000F, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: renamable $vgpr4 = V_MOV_B32_e32 0, implicit $exec
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.7.Flow:
; CHECK-NEXT: successors: %bb.8(0x40000000), %bb.9(0x40000000)
- ; CHECK-NEXT: liveins: $vgpr0, $vgpr4, $sgpr0_sgpr1, $sgpr2_sgpr3, $sgpr6_sgpr7, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr4, $sgpr2_sgpr3, $sgpr6_sgpr7, $sgpr0_sgpr1_sgpr2_sgpr3:0x000000000000000F, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: renamable $sgpr4_sgpr5 = S_MOV_B64 -1
; CHECK-NEXT: $vcc = S_ANDN2_B64 $exec, killed renamable $sgpr6_sgpr7, implicit-def dead $scc
@@ -121,7 +120,7 @@ define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1) #0 {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: bb.8.bb11:
; CHECK-NEXT: successors: %bb.9(0x80000000)
- ; CHECK-NEXT: liveins: $vgpr0, $vgpr4, $sgpr0_sgpr1, $sgpr2_sgpr3, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
+ ; CHECK-NEXT: liveins: $vgpr0, $vgpr4, $sgpr2_sgpr3, $sgpr0_sgpr1_sgpr2_sgpr3:0x000000000000000F, $vgpr0_vgpr1:0x000000000000000C, $vgpr2_vgpr3, $sgpr20_sgpr21_sgpr22_sgpr23
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: renamable $sgpr4_sgpr5 = S_MOV_B64 0
; CHECK-NEXT: $sgpr6_sgpr7 = S_MOV_B64 $sgpr2_sgpr3
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
index ea406781a4e79..b1cd696b145f3 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_dynelt.ll
@@ -62,14 +62,14 @@ entry:
define amdgpu_kernel void @float4_inselt_undef(ptr addrspace(1) %out, i32 %sel) {
; GCN-LABEL: float4_inselt_undef:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 1.0
; GCN-NEXT: v_mov_b32_e32 v1, v0
; GCN-NEXT: v_mov_b32_e32 v2, v0
; GCN-NEXT: v_mov_b32_e32 v3, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v5, s1
; GCN-NEXT: v_mov_b32_e32 v4, s0
+; GCN-NEXT: v_mov_b32_e32 v5, s1
; GCN-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NEXT: s_endpgm
;
@@ -156,20 +156,20 @@ entry:
define amdgpu_kernel void @float2_inselt(ptr addrspace(1) %out, <2 x float> %vec, i32 %sel) {
; GCN-LABEL: float2_inselt:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_load_dword s4, s[4:5], 0x34
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_lg_u32 s2, 1
-; GCN-NEXT: v_mov_b32_e32 v0, s1
+; GCN-NEXT: v_mov_b32_e32 v2, s3
+; GCN-NEXT: s_cmp_lg_u32 s4, 1
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
-; GCN-NEXT: s_cmp_lg_u32 s2, 0
-; GCN-NEXT: v_cndmask_b32_e32 v1, 1.0, v0, vcc
-; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cmp_lg_u32 s4, 0
+; GCN-NEXT: v_cndmask_b32_e32 v3, 1.0, v2, vcc
+; GCN-NEXT: v_mov_b32_e32 v2, s2
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
-; GCN-NEXT: v_mov_b32_e32 v2, s4
-; GCN-NEXT: v_cndmask_b32_e32 v0, 1.0, v0, vcc
-; GCN-NEXT: v_mov_b32_e32 v3, s5
-; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_cndmask_b32_e32 v2, 1.0, v2, vcc
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN-NEXT: s_endpgm
;
; GCN-O0-LABEL: float2_inselt:
@@ -816,19 +816,19 @@ define amdgpu_kernel void @half4_inselt(ptr addrspace(1) %out, <4 x half> %vec,
; GCN-LABEL: half4_inselt:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x34
+; GCN-NEXT: s_load_dword s5, s[4:5], 0x34
; GCN-NEXT: s_mov_b32 s4, 0x3c003c00
-; GCN-NEXT: s_mov_b32 s5, s4
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_xor_b64 s[4:5], s[2:3], s[4:5]
-; GCN-NEXT: s_lshl_b32 s6, s6, 4
-; GCN-NEXT: s_lshl_b64 s[6:7], 0xffff, s6
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
-; GCN-NEXT: s_xor_b64 s[2:3], s[4:5], s[2:3]
; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_lshl_b32 s0, s5, 4
+; GCN-NEXT: s_mov_b32 s5, s4
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v3, s3
+; GCN-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
+; GCN-NEXT: s_xor_b64 s[4:5], s[2:3], s[4:5]
+; GCN-NEXT: s_and_b64 s[0:1], s[4:5], s[0:1]
+; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN-NEXT: s_endpgm
;
@@ -872,14 +872,14 @@ define amdgpu_kernel void @half2_inselt(ptr addrspace(1) %out, <2 x half> %vec,
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 s3, s3, 4
-; GCN-NEXT: s_xor_b32 s4, s2, 0x3c003c00
-; GCN-NEXT: s_lshl_b32 s3, 0xffff, s3
-; GCN-NEXT: s_and_b32 s3, s4, s3
-; GCN-NEXT: s_xor_b32 s2, s3, s2
-; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_lshl_b32 s1, s3, 4
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_xor_b32 s0, s2, 0x3c003c00
+; GCN-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GCN-NEXT: s_and_b32 s0, s0, s1
+; GCN-NEXT: s_xor_b32 s0, s0, s2
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
;
@@ -1038,14 +1038,14 @@ define amdgpu_kernel void @short2_inselt(ptr addrspace(1) %out, <2 x i16> %vec,
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 s3, s3, 4
-; GCN-NEXT: s_xor_b32 s4, s2, 0x10001
-; GCN-NEXT: s_lshl_b32 s3, 0xffff, s3
-; GCN-NEXT: s_and_b32 s3, s4, s3
-; GCN-NEXT: s_xor_b32 s2, s3, s2
-; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_lshl_b32 s1, s3, 4
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_xor_b32 s0, s2, 0x10001
+; GCN-NEXT: s_lshl_b32 s1, 0xffff, s1
+; GCN-NEXT: s_and_b32 s0, s0, s1
+; GCN-NEXT: s_xor_b32 s0, s0, s2
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
;
@@ -1084,19 +1084,19 @@ define amdgpu_kernel void @short4_inselt(ptr addrspace(1) %out, <4 x i16> %vec,
; GCN-LABEL: short4_inselt:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x34
+; GCN-NEXT: s_load_dword s5, s[4:5], 0x34
; GCN-NEXT: s_mov_b32 s4, 0x10001
-; GCN-NEXT: s_mov_b32 s5, s4
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_xor_b64 s[4:5], s[2:3], s[4:5]
-; GCN-NEXT: s_lshl_b32 s6, s6, 4
-; GCN-NEXT: s_lshl_b64 s[6:7], 0xffff, s6
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
-; GCN-NEXT: s_xor_b64 s[2:3], s[4:5], s[2:3]
; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_lshl_b32 s0, s5, 4
+; GCN-NEXT: s_mov_b32 s5, s4
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v3, s3
+; GCN-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
+; GCN-NEXT: s_xor_b64 s[4:5], s[2:3], s[4:5]
+; GCN-NEXT: s_and_b64 s[0:1], s[4:5], s[0:1]
+; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN-NEXT: s_endpgm
;
@@ -1139,18 +1139,18 @@ define amdgpu_kernel void @byte8_inselt(ptr addrspace(1) %out, <8 x i8> %vec, i3
; GCN-LABEL: byte8_inselt:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x34
+; GCN-NEXT: s_load_dword s4, s[4:5], 0x34
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_xor_b32 s5, s3, 0x1010101
-; GCN-NEXT: s_lshl_b32 s6, s6, 3
-; GCN-NEXT: s_xor_b32 s4, s2, 0x1010101
-; GCN-NEXT: s_lshl_b64 s[6:7], 0xff, s6
-; GCN-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
-; GCN-NEXT: s_xor_b64 s[2:3], s[4:5], s[2:3]
; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_lshl_b32 s4, s4, 3
; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v3, s3
+; GCN-NEXT: s_xor_b32 s1, s3, 0x1010101
+; GCN-NEXT: s_xor_b32 s0, s2, 0x1010101
+; GCN-NEXT: s_lshl_b64 s[4:5], 0xff, s4
+; GCN-NEXT: s_and_b64 s[0:1], s[0:1], s[4:5]
+; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GCN-NEXT: s_endpgm
;
@@ -3883,26 +3883,26 @@ define amdgpu_kernel void @bit4_inselt(ptr addrspace(1) %out, <4 x i1> %vec, i32
; GCN-NEXT: buffer_store_byte v0, off, s[12:15], 0 offset:1
; GCN-NEXT: v_mov_b32_e32 v0, 1
; GCN-NEXT: buffer_store_byte v0, v1, s[12:15], 0 offen
-; GCN-NEXT: buffer_load_ubyte v0, off, s[12:15], 0
-; GCN-NEXT: buffer_load_ubyte v1, off, s[12:15], 0 offset:1
-; GCN-NEXT: buffer_load_ubyte v2, off, s[12:15], 0 offset:2
-; GCN-NEXT: buffer_load_ubyte v3, off, s[12:15], 0 offset:3
+; GCN-NEXT: buffer_load_ubyte v2, off, s[12:15], 0
+; GCN-NEXT: buffer_load_ubyte v3, off, s[12:15], 0 offset:1
+; GCN-NEXT: buffer_load_ubyte v4, off, s[12:15], 0 offset:2
+; GCN-NEXT: buffer_load_ubyte v5, off, s[12:15], 0 offset:3
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
; GCN-NEXT: s_waitcnt vmcnt(3)
-; GCN-NEXT: v_and_b32_e32 v0, 1, v0
+; GCN-NEXT: v_and_b32_e32 v2, 1, v2
; GCN-NEXT: s_waitcnt vmcnt(2)
-; GCN-NEXT: v_and_b32_e32 v1, 1, v1
+; GCN-NEXT: v_and_b32_e32 v3, 1, v3
; GCN-NEXT: s_waitcnt vmcnt(1)
-; GCN-NEXT: v_and_b32_e32 v2, 1, v2
-; GCN-NEXT: v_lshlrev_b16_e32 v1, 1, v1
-; GCN-NEXT: v_lshlrev_b16_e32 v2, 2, v2
-; GCN-NEXT: v_or_b32_e32 v0, v0, v1
+; GCN-NEXT: v_and_b32_e32 v4, 1, v4
+; GCN-NEXT: v_lshlrev_b16_e32 v3, 1, v3
+; GCN-NEXT: v_lshlrev_b16_e32 v4, 2, v4
+; GCN-NEXT: v_or_b32_e32 v2, v2, v3
+; GCN-NEXT: v_or_b32_e32 v2, v2, v4
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_lshlrev_b16_e32 v3, 3, v3
-; GCN-NEXT: v_or_b32_e32 v0, v0, v2
-; GCN-NEXT: v_or_b32_e32 v0, v0, v3
-; GCN-NEXT: v_and_b32_e32 v2, 15, v0
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_lshlrev_b16_e32 v3, 3, v5
+; GCN-NEXT: v_or_b32_e32 v2, v2, v3
+; GCN-NEXT: v_and_b32_e32 v2, 15, v2
; GCN-NEXT: flat_store_byte v[0:1], v2
; GCN-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
index 7cbf9aeacfe48..1f3b3c9fe146c 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.ll
@@ -497,38 +497,42 @@ define <12 x float> @insertelement_to_v12f32_undef() nounwind {
define amdgpu_kernel void @dynamic_insertelement_v2f32(ptr addrspace(1) %out, <2 x float> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v2f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; SI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; SI-NEXT: s_load_dword s8, s[8:9], 0x4
; SI-NEXT: v_mov_b32_e32 v0, 0x40a00000
; SI-NEXT: s_mov_b32 s7, 0x100f000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_lg_u32 s2, 1
-; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: v_mov_b32_e32 v1, s3
+; SI-NEXT: s_cmp_lg_u32 s8, 1
; SI-NEXT: s_cselect_b64 vcc, -1, 0
-; SI-NEXT: s_cmp_lg_u32 s2, 0
+; SI-NEXT: s_cmp_lg_u32 s8, 0
; SI-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc
-; SI-NEXT: v_mov_b32_e32 v2, s0
+; SI-NEXT: v_mov_b32_e32 v2, s2
; SI-NEXT: s_cselect_b64 vcc, -1, 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v2f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT: s_load_dword s8, s[8:9], 0x10
; VI-NEXT: v_mov_b32_e32 v0, 0x40a00000
; VI-NEXT: s_mov_b32 s7, 0x1100f000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_lg_u32 s2, 1
-; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v1, s3
+; VI-NEXT: s_cmp_lg_u32 s8, 1
; VI-NEXT: s_cselect_b64 vcc, -1, 0
-; VI-NEXT: s_cmp_lg_u32 s2, 0
+; VI-NEXT: s_cmp_lg_u32 s8, 0
; VI-NEXT: v_cndmask_b32_e32 v1, v0, v1, vcc
-; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
@@ -1019,33 +1023,37 @@ define amdgpu_kernel void @dynamic_insertelement_v16f32(ptr addrspace(1) %out, <
define amdgpu_kernel void @dynamic_insertelement_v2i32(ptr addrspace(1) %out, <2 x i32> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v2i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; SI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; SI-NEXT: s_load_dword s8, s[8:9], 0x4
; SI-NEXT: s_mov_b32 s7, 0x100f000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_lg_u32 s2, 1
-; SI-NEXT: s_cselect_b32 s1, s1, 5
-; SI-NEXT: s_cmp_lg_u32 s2, 0
-; SI-NEXT: s_cselect_b32 s0, s0, 5
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_cmp_lg_u32 s8, 1
+; SI-NEXT: s_cselect_b32 s0, s3, 5
+; SI-NEXT: s_cmp_lg_u32 s8, 0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_cselect_b32 s1, s2, 5
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: v_mov_b32_e32 v1, s0
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v2i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT: s_load_dword s8, s[8:9], 0x10
; VI-NEXT: s_mov_b32 s7, 0x1100f000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_lg_u32 s2, 1
-; VI-NEXT: s_cselect_b32 s1, s1, 5
-; VI-NEXT: s_cmp_lg_u32 s2, 0
-; VI-NEXT: s_cselect_b32 s0, s0, 5
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_cmp_lg_u32 s8, 1
+; VI-NEXT: s_cselect_b32 s0, s3, 5
+; VI-NEXT: s_cmp_lg_u32 s8, 0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_cselect_b32 s1, s2, 5
+; VI-NEXT: v_mov_b32_e32 v0, s1
+; VI-NEXT: v_mov_b32_e32 v1, s0
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
%vecins = insertelement <2 x i32> %a, i32 5, i32 %b
@@ -1508,33 +1516,35 @@ define amdgpu_kernel void @dynamic_insertelement_v2i16(ptr addrspace(1) %out, <2
; SI-LABEL: dynamic_insertelement_v2i16:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; SI-NEXT: s_mov_b32 s7, 0x100f000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_lshl_b32 s5, s5, 4
-; SI-NEXT: s_xor_b32 s6, s4, 0x50005
-; SI-NEXT: s_lshl_b32 s5, 0xffff, s5
-; SI-NEXT: s_and_b32 s5, s6, s5
-; SI-NEXT: s_xor_b32 s4, s5, s4
-; SI-NEXT: s_mov_b32 s3, 0x100f000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_lshl_b32 s1, s3, 4
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_xor_b32 s0, s2, 0x50005
+; SI-NEXT: s_lshl_b32 s1, 0xffff, s1
+; SI-NEXT: s_and_b32 s0, s0, s1
+; SI-NEXT: s_xor_b32 s0, s0, s2
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v2i16:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT: s_mov_b32 s7, 0x1100f000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: s_lshl_b32 s5, s5, 4
-; VI-NEXT: s_xor_b32 s6, s4, 0x50005
-; VI-NEXT: s_lshl_b32 s5, 0xffff, s5
-; VI-NEXT: s_and_b32 s5, s6, s5
-; VI-NEXT: s_xor_b32 s4, s5, s4
-; VI-NEXT: s_mov_b32 s3, 0x1100f000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_lshl_b32 s1, s3, 4
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_xor_b32 s0, s2, 0x50005
+; VI-NEXT: s_lshl_b32 s1, 0xffff, s1
+; VI-NEXT: s_and_b32 s0, s0, s1
+; VI-NEXT: s_xor_b32 s0, s0, s2
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
%vecins = insertelement <2 x i16> %a, i16 5, i32 %b
store <2 x i16> %vecins, ptr addrspace(1) %out, align 8
@@ -1571,13 +1581,13 @@ define amdgpu_kernel void @dynamic_insertelement_v3i16(ptr addrspace(1) %out, <3
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 s4, s0
-; VI-NEXT: s_mov_b32 s0, 0x50005
+; VI-NEXT: s_lshl_b32 s0, s8, 4
+; VI-NEXT: s_mov_b32 s8, 0x50005
+; VI-NEXT: s_mov_b32 s9, s8
; VI-NEXT: s_mov_b32 s5, s1
-; VI-NEXT: s_mov_b32 s1, s0
-; VI-NEXT: s_lshl_b32 s8, s8, 4
-; VI-NEXT: s_xor_b64 s[0:1], s[2:3], s[0:1]
-; VI-NEXT: s_lshl_b64 s[8:9], 0xffff, s8
-; VI-NEXT: s_and_b64 s[0:1], s[0:1], s[8:9]
+; VI-NEXT: s_lshl_b64 s[0:1], 0xffff, s0
+; VI-NEXT: s_xor_b64 s[8:9], s[2:3], s[8:9]
+; VI-NEXT: s_and_b64 s[0:1], s[8:9], s[0:1]
; VI-NEXT: s_xor_b64 s[0:1], s[0:1], s[2:3]
; VI-NEXT: v_mov_b32_e32 v0, s1
; VI-NEXT: buffer_store_short v0, off, s[4:7], 0 offset:4
@@ -2131,58 +2141,56 @@ define amdgpu_kernel void @dynamic_insertelement_v2i64(ptr addrspace(1) %out, <2
define amdgpu_kernel void @dynamic_insertelement_v3i64(ptr addrspace(1) %out, <3 x i64> %a, i32 %b) nounwind {
; SI-LABEL: dynamic_insertelement_v3i64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s10, s[8:9], 0x10
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x8
-; SI-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0xc
-; SI-NEXT: s_mov_b32 s3, 0x100f000
+; SI-NEXT: s_load_dword s12, s[8:9], 0x10
+; SI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x8
+; SI-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x0
+; SI-NEXT: s_mov_b32 s11, 0x100f000
+; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_eq_u32 s10, 1
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_cselect_b32 s7, 0, s7
-; SI-NEXT: s_cselect_b32 s6, 5, s6
-; SI-NEXT: s_cmp_eq_u32 s10, 0
+; SI-NEXT: s_cmp_eq_u32 s12, 1
+; SI-NEXT: s_cselect_b32 s3, 0, s3
+; SI-NEXT: s_cselect_b32 s2, 5, s2
+; SI-NEXT: s_cmp_eq_u32 s12, 0
+; SI-NEXT: s_cselect_b32 s1, 0, s1
+; SI-NEXT: s_cselect_b32 s0, 5, s0
+; SI-NEXT: s_cmp_eq_u32 s12, 2
; SI-NEXT: s_cselect_b32 s5, 0, s5
; SI-NEXT: s_cselect_b32 s4, 5, s4
-; SI-NEXT: s_cmp_eq_u32 s10, 2
-; SI-NEXT: s_cselect_b32 s9, 0, s9
-; SI-NEXT: s_cselect_b32 s8, 5, s8
-; SI-NEXT: v_mov_b32_e32 v0, s8
-; SI-NEXT: v_mov_b32_e32 v1, s9
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 offset:16
; SI-NEXT: v_mov_b32_e32 v0, s4
; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: v_mov_b32_e32 v2, s6
-; SI-NEXT: v_mov_b32_e32 v3, s7
-; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 offset:16
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_mov_b32_e32 v3, s3
+; SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: dynamic_insertelement_v3i64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s10, s[8:9], 0x40
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x20
-; VI-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x30
-; VI-NEXT: s_mov_b32 s3, 0x1100f000
+; VI-NEXT: s_load_dword s12, s[8:9], 0x40
+; VI-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x20
+; VI-NEXT: s_load_dwordx2 s[8:9], s[8:9], 0x0
+; VI-NEXT: s_mov_b32 s11, 0x1100f000
+; VI-NEXT: s_mov_b32 s10, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s10, 1
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: s_cselect_b32 s7, 0, s7
-; VI-NEXT: s_cselect_b32 s6, 5, s6
-; VI-NEXT: s_cmp_eq_u32 s10, 0
+; VI-NEXT: s_cmp_eq_u32 s12, 1
+; VI-NEXT: s_cselect_b32 s3, 0, s3
+; VI-NEXT: s_cselect_b32 s2, 5, s2
+; VI-NEXT: s_cmp_eq_u32 s12, 0
+; VI-NEXT: s_cselect_b32 s1, 0, s1
+; VI-NEXT: s_cselect_b32 s0, 5, s0
+; VI-NEXT: s_cmp_eq_u32 s12, 2
; VI-NEXT: s_cselect_b32 s5, 0, s5
; VI-NEXT: s_cselect_b32 s4, 5, s4
-; VI-NEXT: s_cmp_eq_u32 s10, 2
-; VI-NEXT: s_cselect_b32 s9, 0, s9
-; VI-NEXT: s_cselect_b32 s8, 5, s8
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: v_mov_b32_e32 v1, s9
-; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0 offset:16
; VI-NEXT: v_mov_b32_e32 v0, s4
; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: v_mov_b32_e32 v2, s6
-; VI-NEXT: v_mov_b32_e32 v3, s7
-; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0 offset:16
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[8:11], 0
; VI-NEXT: s_endpgm
%vecins = insertelement <3 x i64> %a, i64 5, i32 %b
store <3 x i64> %vecins, ptr addrspace(1) %out, align 32
diff --git a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
index 33424b7f0d16e..c652d4193032e 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_waitcnt_for_precise_memory.ll
@@ -882,26 +882,24 @@ define void @flat_atomic_xchg_i32_noret(ptr %ptr, i32 %in) {
define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrspace(3) %local) {
; GFX9-LABEL: atomic_add_ret_local:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_mov_b64 s[6:7], exec
+; GFX9-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-NEXT: ; implicit-def: $vgpr1
-; GFX9-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-NEXT: s_cbranch_execz .LBB7_2
; GFX9-NEXT: ; %bb.1:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-NEXT: s_mul_i32 s2, s2, 5
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX9-NEXT: s_mul_i32 s3, s3, 5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: .LBB7_2:
-; GFX9-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: v_mad_u32_u24 v0, 5, v0, s2
@@ -911,26 +909,24 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX90A-LABEL: atomic_add_ret_local:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_mov_b64 s[2:3], exec
-; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b64 s[6:7], exec
+; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: ; implicit-def: $vgpr1
-; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX90A-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB7_2
; GFX90A-NEXT: ; %bb.1:
-; GFX90A-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX90A-NEXT: s_mul_i32 s2, s2, 5
-; GFX90A-NEXT: v_mov_b32_e32 v2, s2
-; GFX90A-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX90A-NEXT: s_mul_i32 s3, s3, 5
+; GFX90A-NEXT: v_mov_b32_e32 v1, s2
+; GFX90A-NEXT: v_mov_b32_e32 v2, s3
; GFX90A-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB7_2:
-; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX90A-NEXT: v_readfirstlane_b32 s2, v1
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
; GFX90A-NEXT: v_mad_u32_u24 v0, 5, v0, s2
@@ -940,27 +936,25 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX10-LABEL: atomic_add_ret_local:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_mov_b32 s1, exec_lo
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_mov_b32 s6, exec_lo
; GFX10-NEXT: ; implicit-def: $vgpr1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
-; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
+; GFX10-NEXT: s_and_saveexec_b32 s3, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB7_2
; GFX10-NEXT: ; %bb.1:
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX10-NEXT: s_mul_i32 s1, s1, 5
-; GFX10-NEXT: v_mov_b32_e32 v2, s1
+; GFX10-NEXT: s_bcnt1_i32_b32 s4, s6
; GFX10-NEXT: v_mov_b32_e32 v1, s2
+; GFX10-NEXT: s_mul_i32 s4, s4, 5
+; GFX10-NEXT: v_mov_b32_e32 v2, s4
; GFX10-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: .LBB7_2:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: v_readfirstlane_b32 s2, v1
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: v_mad_u32_u24 v0, 5, v0, s2
@@ -970,26 +964,24 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX9-FLATSCR-LABEL: atomic_add_ret_local:
; GFX9-FLATSCR: ; %bb.0:
-; GFX9-FLATSCR-NEXT: s_mov_b64 s[2:3], exec
-; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
-; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, s3, v0
+; GFX9-FLATSCR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-FLATSCR-NEXT: s_mov_b64 s[6:7], exec
+; GFX9-FLATSCR-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX9-FLATSCR-NEXT: v_mbcnt_hi_u32_b32 v0, s7, v0
; GFX9-FLATSCR-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-FLATSCR-NEXT: ; implicit-def: $vgpr1
-; GFX9-FLATSCR-NEXT: s_and_saveexec_b64 s[0:1], vcc
+; GFX9-FLATSCR-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-FLATSCR-NEXT: s_cbranch_execz .LBB7_2
; GFX9-FLATSCR-NEXT: ; %bb.1:
-; GFX9-FLATSCR-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9-FLATSCR-NEXT: s_mul_i32 s2, s2, 5
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-FLATSCR-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX9-FLATSCR-NEXT: s_mul_i32 s3, s3, 5
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, s3
; GFX9-FLATSCR-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-FLATSCR-NEXT: .LBB7_2:
-; GFX9-FLATSCR-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9-FLATSCR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-FLATSCR-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-FLATSCR-NEXT: v_readfirstlane_b32 s2, v1
; GFX9-FLATSCR-NEXT: v_mov_b32_e32 v2, 0
; GFX9-FLATSCR-NEXT: v_mad_u32_u24 v0, 5, v0, s2
@@ -999,27 +991,25 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX11-LABEL: atomic_add_ret_local:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_mov_b32 s1, exec_lo
-; GFX11-NEXT: s_mov_b32 s0, exec_lo
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s6, exec_lo
+; GFX11-NEXT: s_mov_b32 s3, exec_lo
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX11-NEXT: s_cbranch_execz .LBB7_2
; GFX11-NEXT: ; %bb.1:
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_bcnt1_i32_b32 s1, s1
+; GFX11-NEXT: s_bcnt1_i32_b32 s4, s6
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: s_mul_i32 s1, s1, 5
-; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: s_mul_i32 s4, s4, 5
+; GFX11-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s4
; GFX11-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: .LBB7_2:
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: v_readfirstlane_b32 s2, v1
; GFX11-NEXT: v_mov_b32_e32 v1, 0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
@@ -1030,35 +1020,32 @@ define amdgpu_kernel void @atomic_add_ret_local(ptr addrspace(1) %out, ptr addrs
;
; GFX12-LABEL: atomic_add_ret_local:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_mov_b32 s1, exec_lo
-; GFX12-NEXT: s_mov_b32 s0, exec_lo
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s1, 0
+; GFX12-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_mov_b32 s6, exec_lo
+; GFX12-NEXT: s_mov_b32 s3, exec_lo
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
; GFX12-NEXT: ; implicit-def: $vgpr1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v0
; GFX12-NEXT: s_cbranch_execz .LBB7_2
; GFX12-NEXT: ; %bb.1:
-; GFX12-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bcnt1_i32_b32 s1, s1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_mul_i32 s1, s1, 5
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: s_bcnt1_i32_b32 s4, s6
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_mul_i32 s4, s4, 5
+; GFX12-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s4
; GFX12-NEXT: ds_add_rtn_u32 v1, v1, v2
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: .LBB7_2:
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX12-NEXT: v_readfirstlane_b32 s2, v1
; GFX12-NEXT: v_mov_b32_e32 v1, 0
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_mad_u32_u24 v0, 5, v0, s2
; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: s_wait_storecnt 0x0
; GFX12-NEXT: s_endpgm
%val = atomicrmw volatile add ptr addrspace(3) %local, i32 5 seq_cst
diff --git a/llvm/test/CodeGen/AMDGPU/kernel-args.ll b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
index ea0a8cbe663fd..ff60bc1d1cd93 100644
--- a/llvm/test/CodeGen/AMDGPU/kernel-args.ll
+++ b/llvm/test/CodeGen/AMDGPU/kernel-args.ll
@@ -8,10 +8,9 @@
define amdgpu_kernel void @i8_arg(ptr addrspace(1) nocapture %out, i8 %in) nounwind {
; SI-LABEL: i8_arg:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_and_b32 s4, s2, 0xff
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -20,20 +19,18 @@ define amdgpu_kernel void @i8_arg(ptr addrspace(1) nocapture %out, i8 %in) nounw
;
; VI-LABEL: i8_arg:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0xff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0xff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: i8_arg:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0xff
@@ -80,10 +77,9 @@ define amdgpu_kernel void @i8_arg(ptr addrspace(1) nocapture %out, i8 %in) nounw
define amdgpu_kernel void @i8_zext_arg(ptr addrspace(1) nocapture %out, i8 zeroext %in) nounwind {
; SI-LABEL: i8_zext_arg:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_and_b32 s4, s2, 0xff
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -92,20 +88,18 @@ define amdgpu_kernel void @i8_zext_arg(ptr addrspace(1) nocapture %out, i8 zeroe
;
; VI-LABEL: i8_zext_arg:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0xff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0xff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: i8_zext_arg:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0xff
@@ -155,10 +149,9 @@ define amdgpu_kernel void @i8_zext_arg(ptr addrspace(1) nocapture %out, i8 zeroe
define amdgpu_kernel void @i8_sext_arg(ptr addrspace(1) nocapture %out, i8 signext %in) nounwind {
; SI-LABEL: i8_sext_arg:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_sext_i32_i8 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -167,20 +160,18 @@ define amdgpu_kernel void @i8_sext_arg(ptr addrspace(1) nocapture %out, i8 signe
;
; VI-LABEL: i8_sext_arg:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_sext_i32_i8 s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_sext_i32_i8 s0, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: i8_sext_arg:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_sext_i32_i8 s2, s2
@@ -230,10 +221,9 @@ define amdgpu_kernel void @i8_sext_arg(ptr addrspace(1) nocapture %out, i8 signe
define amdgpu_kernel void @i16_arg(ptr addrspace(1) nocapture %out, i16 %in) nounwind {
; SI-LABEL: i16_arg:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_and_b32 s4, s2, 0xffff
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -242,20 +232,18 @@ define amdgpu_kernel void @i16_arg(ptr addrspace(1) nocapture %out, i16 %in) nou
;
; VI-LABEL: i16_arg:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0xffff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0xffff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: i16_arg:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
@@ -302,10 +290,9 @@ define amdgpu_kernel void @i16_arg(ptr addrspace(1) nocapture %out, i16 %in) nou
define amdgpu_kernel void @i16_zext_arg(ptr addrspace(1) nocapture %out, i16 zeroext %in) nounwind {
; SI-LABEL: i16_zext_arg:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_and_b32 s4, s2, 0xffff
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -314,20 +301,18 @@ define amdgpu_kernel void @i16_zext_arg(ptr addrspace(1) nocapture %out, i16 zer
;
; VI-LABEL: i16_zext_arg:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0xffff
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 0xffff
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: i16_zext_arg:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 0xffff
@@ -377,10 +362,9 @@ define amdgpu_kernel void @i16_zext_arg(ptr addrspace(1) nocapture %out, i16 zer
define amdgpu_kernel void @i16_sext_arg(ptr addrspace(1) nocapture %out, i16 signext %in) nounwind {
; SI-LABEL: i16_sext_arg:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_sext_i32_i16 s4, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -389,20 +373,18 @@ define amdgpu_kernel void @i16_sext_arg(ptr addrspace(1) nocapture %out, i16 sig
;
; VI-LABEL: i16_sext_arg:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_sext_i32_i16 s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_sext_i32_i16 s0, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: i16_sext_arg:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_sext_i32_i16 s2, s2
@@ -452,19 +434,19 @@ define amdgpu_kernel void @i16_sext_arg(ptr addrspace(1) nocapture %out, i16 sig
define amdgpu_kernel void @i32_arg(ptr addrspace(1) nocapture %out, i32 %in) nounwind {
; SI-LABEL: i32_arg:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: i32_arg:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -474,8 +456,7 @@ define amdgpu_kernel void @i32_arg(ptr addrspace(1) nocapture %out, i32 %in) nou
;
; GFX9-LABEL: i32_arg:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -511,19 +492,19 @@ entry:
define amdgpu_kernel void @f32_arg(ptr addrspace(1) nocapture %out, float %in) nounwind {
; SI-LABEL: f32_arg:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: f32_arg:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -533,8 +514,7 @@ define amdgpu_kernel void @f32_arg(ptr addrspace(1) nocapture %out, float %in) n
;
; GFX9-LABEL: f32_arg:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -570,19 +550,19 @@ entry:
define amdgpu_kernel void @v2i8_arg(ptr addrspace(1) %out, <2 x i8> %in) {
; SI-LABEL: v2i8_arg:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v2i8_arg:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -592,8 +572,7 @@ define amdgpu_kernel void @v2i8_arg(ptr addrspace(1) %out, <2 x i8> %in) {
;
; GFX9-LABEL: v2i8_arg:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -659,19 +638,19 @@ entry:
define amdgpu_kernel void @v2i16_arg(ptr addrspace(1) %out, <2 x i16> %in) {
; SI-LABEL: v2i16_arg:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v2i16_arg:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -681,8 +660,7 @@ define amdgpu_kernel void @v2i16_arg(ptr addrspace(1) %out, <2 x i16> %in) {
;
; GFX9-LABEL: v2i16_arg:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -734,8 +712,8 @@ define amdgpu_kernel void @v2i32_arg(ptr addrspace(1) nocapture %out, <2 x i32>
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
@@ -797,8 +775,8 @@ define amdgpu_kernel void @v2f32_arg(ptr addrspace(1) nocapture %out, <2 x float
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
@@ -844,40 +822,39 @@ entry:
define amdgpu_kernel void @v3i8_arg(ptr addrspace(1) nocapture %out, <3 x i8> %in) nounwind {
; SI-LABEL: v3i8_arg:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshr_b32 s4, s6, 16
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: buffer_store_byte v1, off, s[0:3], 0 offset:2
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_lshr_b32 s3, s2, 16
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s3
+; SI-NEXT: buffer_store_byte v1, off, s[4:7], 0 offset:2
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v3i8_arg:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s3, s2, 16
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_lshr_b32 s2, s2, 16
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: s_add_u32 s0, s0, 2
+; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_addc_u32 s1, s1, 0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v5, s3
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_mov_b32_e32 v4, s2
-; VI-NEXT: flat_store_byte v[2:3], v5
-; VI-NEXT: flat_store_short v[0:1], v4
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: flat_store_byte v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: v3i8_arg:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -1000,16 +977,16 @@ define amdgpu_kernel void @v3i16_arg(ptr addrspace(1) nocapture %out, <3 x i16>
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_u32 s4, s0, 4
-; VI-NEXT: s_addc_u32 s5, s1, 0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v4, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v5, s2
-; VI-NEXT: flat_store_short v[2:3], v4
-; VI-NEXT: flat_store_dword v[0:1], v5
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_add_u32 s0, s0, 4
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_addc_u32 s1, s1, 0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s3
+; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: v3i16_arg:
@@ -1252,19 +1229,19 @@ entry:
define amdgpu_kernel void @v4i8_arg(ptr addrspace(1) %out, <4 x i8> %in) {
; SI-LABEL: v4i8_arg:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v4i8_arg:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -1274,8 +1251,7 @@ define amdgpu_kernel void @v4i8_arg(ptr addrspace(1) %out, <4 x i8> %in) {
;
; GFX9-LABEL: v4i8_arg:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -1327,8 +1303,8 @@ define amdgpu_kernel void @v4i16_arg(ptr addrspace(1) %out, <4 x i16> %in) {
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
@@ -1523,32 +1499,59 @@ define amdgpu_kernel void @v5i8_arg(ptr addrspace(1) nocapture %out, <5 x i8> %i
; SI-LABEL: v5i8_arg:
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s4, 0
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, s0
-; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_and_b32 s5, s1, 0xff000000
+; SI-NEXT: s_bfe_u32 s8, s1, 0x80010
+; SI-NEXT: s_and_b32 s9, s1, 0xffff
+; SI-NEXT: s_mov_b32 s1, s4
; SI-NEXT: v_mov_b32_e32 v0, s3
+; SI-NEXT: v_mov_b32_e32 v1, s2
+; SI-NEXT: s_lshl_b32 s2, s8, 16
+; SI-NEXT: s_or_b32 s2, s5, s2
+; SI-NEXT: s_or_b32 s5, s9, s2
+; SI-NEXT: s_or_b64 s[4:5], s[0:1], s[4:5]
; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0 offset:4
-; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s2
-; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: buffer_store_dword v1, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v5i8_arg:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_add_u32 s4, s0, 4
-; VI-NEXT: s_addc_u32 s5, s1, 0
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v4, s3
+; VI-NEXT: s_lshr_b32 s5, s1, 24
+; VI-NEXT: s_and_b32 s4, s1, 0xffff
+; VI-NEXT: s_bfe_u32 s1, s1, 0x80010
+; VI-NEXT: s_lshl_b32 s5, s5, 8
+; VI-NEXT: s_or_b32 s1, s1, s5
+; VI-NEXT: s_lshr_b32 s7, s0, 24
+; VI-NEXT: s_lshl_b32 s1, s1, 16
+; VI-NEXT: s_and_b32 s6, s0, 0xffff
+; VI-NEXT: s_or_b32 s1, s4, s1
+; VI-NEXT: s_lshl_b32 s4, s7, 8
+; VI-NEXT: s_bfe_u32 s0, s0, 0x80010
+; VI-NEXT: s_lshr_b32 s5, s2, 24
+; VI-NEXT: s_or_b32 s0, s0, s4
+; VI-NEXT: s_and_b32 s4, s2, 0xffff
+; VI-NEXT: s_lshl_b32 s5, s5, 8
+; VI-NEXT: s_bfe_u32 s2, s2, 0x80010
+; VI-NEXT: s_lshl_b32 s0, s0, 16
+; VI-NEXT: s_or_b32 s2, s2, s5
+; VI-NEXT: s_or_b32 s0, s6, s0
+; VI-NEXT: s_lshl_b32 s2, s2, 16
+; VI-NEXT: s_or_b32 s2, s4, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v3, s5
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v5, s2
-; VI-NEXT: flat_store_byte v[2:3], v4
-; VI-NEXT: flat_store_dword v[0:1], v5
+; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_add_u32 s0, s0, 4
+; VI-NEXT: flat_store_dword v[0:1], v2
+; VI-NEXT: s_addc_u32 s1, s1, 0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s3
+; VI-NEXT: flat_store_byte v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: v5i8_arg:
@@ -1556,10 +1559,34 @@ define amdgpu_kernel void @v5i8_arg(ptr addrspace(1) nocapture %out, <5 x i8> %i
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_and_b32 s4, s1, 0xffffff00
+; GFX9-NEXT: s_and_b32 s5, s1, 0xff
+; GFX9-NEXT: s_lshr_b32 s6, s1, 24
+; GFX9-NEXT: s_bfe_u32 s1, s1, 0x80010
+; GFX9-NEXT: s_or_b32 s4, s5, s4
+; GFX9-NEXT: s_lshl_b32 s5, s6, 8
+; GFX9-NEXT: s_or_b32 s1, s1, s5
+; GFX9-NEXT: s_and_b32 s4, s4, 0xffff
+; GFX9-NEXT: s_lshl_b32 s1, s1, 16
+; GFX9-NEXT: s_lshr_b32 s5, s0, 24
+; GFX9-NEXT: s_or_b32 s1, s4, s1
+; GFX9-NEXT: s_and_b32 s4, s0, 0xffff
+; GFX9-NEXT: s_lshl_b32 s5, s5, 8
+; GFX9-NEXT: s_bfe_u32 s0, s0, 0x80010
+; GFX9-NEXT: s_or_b32 s0, s0, s5
+; GFX9-NEXT: s_lshl_b32 s0, s0, 16
+; GFX9-NEXT: s_lshr_b32 s5, s2, 24
+; GFX9-NEXT: s_or_b32 s0, s4, s0
+; GFX9-NEXT: s_and_b32 s4, s2, 0xffff
+; GFX9-NEXT: s_lshl_b32 s5, s5, 8
+; GFX9-NEXT: s_bfe_u32 s2, s2, 0x80010
+; GFX9-NEXT: s_or_b32 s2, s2, s5
+; GFX9-NEXT: s_lshl_b32 s2, s2, 16
+; GFX9-NEXT: s_or_b32 s2, s4, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: global_store_byte v0, v1, s[0:1] offset:4
-; GFX9-NEXT: global_store_dword v0, v2, s[0:1]
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: v5i8_arg:
@@ -2370,8 +2397,8 @@ define amdgpu_kernel void @v8i8_arg(ptr addrspace(1) %out, <8 x i8> %in) {
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v3, s3
; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
@@ -4577,10 +4604,9 @@ entry:
define amdgpu_kernel void @i1_arg(ptr addrspace(1) %out, i1 %x) nounwind {
; SI-LABEL: i1_arg:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_and_b32 s4, s2, 1
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -4589,20 +4615,18 @@ define amdgpu_kernel void @i1_arg(ptr addrspace(1) %out, i1 %x) nounwind {
;
; VI-LABEL: i1_arg:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 1
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 1
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_byte v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: i1_arg:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 1
@@ -4668,10 +4692,9 @@ define amdgpu_kernel void @i1_arg(ptr addrspace(1) %out, i1 %x) nounwind {
define amdgpu_kernel void @i1_arg_zext_i32(ptr addrspace(1) %out, i1 %x) nounwind {
; SI-LABEL: i1_arg_zext_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_and_b32 s4, s2, 1
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -4680,20 +4703,18 @@ define amdgpu_kernel void @i1_arg_zext_i32(ptr addrspace(1) %out, i1 %x) nounwin
;
; VI-LABEL: i1_arg_zext_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 1
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 1
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: i1_arg_zext_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 1
@@ -4740,34 +4761,33 @@ define amdgpu_kernel void @i1_arg_zext_i32(ptr addrspace(1) %out, i1 %x) nounwin
define amdgpu_kernel void @i1_arg_zext_i64(ptr addrspace(1) %out, i1 %x) nounwind {
; SI-LABEL: i1_arg_zext_i64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 1
+; SI-NEXT: s_and_b32 s2, s2, 1
; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: i1_arg_zext_i64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_mov_b32_e32 v1, 0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: v_mov_b32_e32 v3, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 1
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 1
+; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: i1_arg_zext_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_and_b32 s2, s2, 1
@@ -4816,10 +4836,9 @@ define amdgpu_kernel void @i1_arg_zext_i64(ptr addrspace(1) %out, i1 %x) nounwin
define amdgpu_kernel void @i1_arg_sext_i32(ptr addrspace(1) %out, i1 %x) nounwind {
; SI-LABEL: i1_arg_sext_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_bfe_i32 s4, s2, 0x10000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -4828,20 +4847,18 @@ define amdgpu_kernel void @i1_arg_sext_i32(ptr addrspace(1) %out, i1 %x) nounwin
;
; VI-LABEL: i1_arg_sext_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfe_i32 s2, s2, 0x10000
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_bfe_i32 s0, s2, 0x10000
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: i1_arg_sext_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_bfe_i32 s2, s2, 0x10000
@@ -4890,10 +4907,9 @@ define amdgpu_kernel void @i1_arg_sext_i32(ptr addrspace(1) %out, i1 %x) nounwin
define amdgpu_kernel void @i1_arg_sext_i64(ptr addrspace(1) %out, i1 %x) nounwind {
; SI-LABEL: i1_arg_sext_i64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_bfe_i64 s[4:5], s[2:3], 0x10000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -4903,27 +4919,25 @@ define amdgpu_kernel void @i1_arg_sext_i64(ptr addrspace(1) %out, i1 %x) nounwin
;
; VI-LABEL: i1_arg_sext_i64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s0, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x10000
-; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_bfe_i64 s[0:1], s[2:3], 0x10000
; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v1, s3
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: i1_arg_sext_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s0, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x10000
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[2:3]
+; GFX9-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: i1_arg_sext_i64:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
index 3dd9252d8e96a..0ff440b021c01 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.alignbyte.ll
@@ -10,64 +10,66 @@ declare i32 @llvm.amdgcn.alignbyte(i32, i32, i32) #0
define amdgpu_kernel void @v_alignbyte_b32(ptr addrspace(1) %out, i32 %src1, i32 %src2, i32 %src3) #1 {
; GCN-LABEL: v_alignbyte_b32:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_load_dword s8, s[4:5], 0xd
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_alignbyte_b32 v0, s0, v0, v1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s3
+; GCN-NEXT: v_mov_b32_e32 v1, s8
+; GCN-NEXT: v_alignbyte_b32 v0, s2, v0, v1
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: v_alignbyte_b32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
-; GFX9-NEXT: v_alignbyte_b32 v1, s0, v1, v2
-; GFX9-NEXT: global_store_dword v0, v1, s[6:7]
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: v_alignbyte_b32 v1, s2, v1, v2
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: v_alignbyte_b32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s2
-; GFX10-NEXT: v_alignbyte_b32 v0, s0, s1, v0
-; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
+; GFX10-NEXT: v_mov_b32_e32 v0, s6
+; GFX10-NEXT: v_alignbyte_b32 v0, s2, s3, v0
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: v_alignbyte_b32:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, s6
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_alignbyte_b32 v0, s0, s1, v0.l
-; GFX11-TRUE16-NEXT: global_store_b32 v1, v0, s[4:5]
+; GFX11-TRUE16-NEXT: v_alignbyte_b32 v0, s2, s3, v0.l
+; GFX11-TRUE16-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: v_alignbyte_b32:
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s6
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_alignbyte_b32 v0, s0, s1, v0
-; GFX11-FAKE16-NEXT: global_store_b32 v1, v0, s[4:5]
+; GFX11-FAKE16-NEXT: v_alignbyte_b32 v0, s2, s3, v0
+; GFX11-FAKE16-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
%val = call i32 @llvm.amdgcn.alignbyte(i32 %src1, i32 %src2, i32 %src3) #0
store i32 %val, ptr addrspace(1) %out
@@ -77,68 +79,58 @@ define amdgpu_kernel void @v_alignbyte_b32(ptr addrspace(1) %out, i32 %src1, i32
define amdgpu_kernel void @v_alignbyte_b32_2(ptr addrspace(1) %out, ptr addrspace(1) %src1, ptr addrspace(1) %src2, i32 %src3) #1 {
; GCN-LABEL: v_alignbyte_b32_2:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; GCN-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GCN-NEXT: s_load_dword s16, s[4:5], 0xf
-; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s11, 0xf000
; GCN-NEXT: s_mov_b32 s14, 0
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v1, 0
-; GCN-NEXT: s_mov_b32 s15, s7
-; GCN-NEXT: s_mov_b64 s[10:11], s[14:15]
+; GCN-NEXT: s_mov_b32 s15, s11
+; GCN-NEXT: s_mov_b64 s[18:19], s[14:15]
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_mov_b64 s[12:13], s[2:3]
+; GCN-NEXT: s_mov_b64 s[16:17], s[4:5]
; GCN-NEXT: buffer_load_dword v2, v[0:1], s[12:15], 0 addr64 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: buffer_load_dword v0, v[0:1], s[8:11], 0 addr64 glc
+; GCN-NEXT: buffer_load_dword v0, v[0:1], s[16:19], 0 addr64 glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: s_mov_b32 s4, s0
-; GCN-NEXT: s_mov_b32 s5, s1
-; GCN-NEXT: v_alignbyte_b32 v0, v2, v0, s16
-; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT: s_mov_b32 s10, -1
+; GCN-NEXT: s_mov_b32 s8, s0
+; GCN-NEXT: s_mov_b32 s9, s1
+; GCN-NEXT: v_alignbyte_b32 v0, v2, v0, s6
+; GCN-NEXT: buffer_store_dword v0, off, s[8:11], 0
; GCN-NEXT: s_endpgm
;
; GFX9-LABEL: v_alignbyte_b32_2:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_load_dword v1, v0, s[2:3] glc
+; GFX9-NEXT: global_load_dword v1, v0, s[10:11] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: global_load_dword v2, v0, s[6:7] glc
+; GFX9-NEXT: global_load_dword v2, v0, s[12:13] glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x3c
; GFX9-NEXT: v_mov_b32_e32 v0, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_alignbyte_b32 v1, v1, v2, s2
-; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
+; GFX9-NEXT: v_alignbyte_b32 v1, v1, v2, s14
+; GFX9-NEXT: global_store_dword v0, v1, s[8:9]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: v_alignbyte_b32_2:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_load_dword v1, v0, s[2:3] glc dlc
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: global_load_dword v0, v0, s[6:7] glc dlc
+; GFX10-NEXT: global_load_dword v0, v0, s[4:5] glc dlc
; GFX10-NEXT: s_waitcnt vmcnt(0)
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x3c
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_alignbyte_b32 v0, v1, v0, s2
+; GFX10-NEXT: v_alignbyte_b32 v0, v1, v0, s6
; GFX10-NEXT: global_store_dword v2, v0, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: v_alignbyte_b32_2:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-TRUE16-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
@@ -146,19 +138,15 @@ define amdgpu_kernel void @v_alignbyte_b32_2(ptr addrspace(1) %out, ptr addrspac
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v1, v0, s[2:3] glc dlc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[6:7] glc dlc
+; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[4:5] glc dlc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x3c
-; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_alignbyte_b32 v0, v1, v0, s2
+; GFX11-TRUE16-NEXT: v_alignbyte_b32 v0, v1, v0, s6
; GFX11-TRUE16-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: v_alignbyte_b32_2:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-FAKE16-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
@@ -166,11 +154,9 @@ define amdgpu_kernel void @v_alignbyte_b32_2(ptr addrspace(1) %out, ptr addrspac
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v1, v0, s[2:3] glc dlc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v0, s[6:7] glc dlc
+; GFX11-FAKE16-NEXT: global_load_b32 v0, v0, s[4:5] glc dlc
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x3c
-; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-FAKE16-NEXT: v_alignbyte_b32 v0, v1, v0, s2
+; GFX11-FAKE16-NEXT: v_alignbyte_b32 v0, v1, v0, s6
; GFX11-FAKE16-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll
index 30e4db85c3f39..1bb9269d898ed 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.dec.ll
@@ -16,12 +16,11 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
; CI-LABEL: lds_atomic_dec_ret_i32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: ds_dec_rtn_u32 v0, v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -31,12 +30,11 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr add
;
; VI-LABEL: lds_atomic_dec_ret_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
-; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: ds_dec_rtn_u32 v0, v1, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
@@ -46,8 +44,7 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr add
;
; GFX9-LABEL: lds_atomic_dec_ret_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -64,12 +61,11 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32(ptr addrspace(1) %out, ptr add
define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
; CI-LABEL: lds_atomic_dec_ret_i32_offset:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -79,12 +75,11 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out,
;
; VI-LABEL: lds_atomic_dec_ret_i32_offset:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
-; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: ds_dec_rtn_u32 v0, v1, v0 offset:16
; VI-NEXT: s_waitcnt lgkmcnt(0)
@@ -94,8 +89,7 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i32_offset(ptr addrspace(1) %out,
;
; GFX9-LABEL: lds_atomic_dec_ret_i32_offset:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -1190,40 +1184,37 @@ define amdgpu_kernel void @atomic_dec_shl_base_lds_0(ptr addrspace(1) %out, ptr
define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
; CI-LABEL: lds_atomic_dec_ret_i64:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_dec_ret_i64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_dec_ret_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -1241,40 +1232,37 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr add
define amdgpu_kernel void @lds_atomic_dec_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
; CI-LABEL: lds_atomic_dec_ret_i64_offset:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_dec_ret_i64_offset:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1] offset:32
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_dec_ret_i64_offset:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll
index 2eeb47dc2f133..54bd028d50673 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.atomic.inc.ll
@@ -16,12 +16,11 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
; CI-LABEL: lds_atomic_inc_ret_i32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: ds_inc_rtn_u32 v0, v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -31,12 +30,11 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr add
;
; VI-LABEL: lds_atomic_inc_ret_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
-; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: ds_inc_rtn_u32 v0, v1, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
@@ -46,8 +44,7 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr add
;
; GFX9-LABEL: lds_atomic_inc_ret_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -64,12 +61,11 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32(ptr addrspace(1) %out, ptr add
define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
; CI-LABEL: lds_atomic_inc_ret_i32_offset:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_mov_b32_e32 v1, s2
; CI-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -79,12 +75,11 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out,
;
; VI-LABEL: lds_atomic_inc_ret_i32_offset:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: s_mov_b32 m0, -1
-; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_mov_b32_e32 v1, s2
; VI-NEXT: ds_inc_rtn_u32 v0, v1, v0 offset:16
; VI-NEXT: s_waitcnt lgkmcnt(0)
@@ -94,8 +89,7 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i32_offset(ptr addrspace(1) %out,
;
; GFX9-LABEL: lds_atomic_inc_ret_i32_offset:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v1, s2
@@ -552,40 +546,37 @@ define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(ptr addrspace(1) %out,
define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
; CI-LABEL: lds_atomic_inc_ret_i64:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_inc_ret_i64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_inc_ret_i64:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
@@ -603,40 +594,37 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr add
define amdgpu_kernel void @lds_atomic_inc_ret_i64_offset(ptr addrspace(1) %out, ptr addrspace(3) %ptr) #0 {
; CI-LABEL: lds_atomic_inc_ret_i64_offset:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: v_mov_b32_e32 v0, 42
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: v_mov_b32_e32 v2, s2
; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; CI-NEXT: s_endpgm
;
; VI-LABEL: lds_atomic_inc_ret_i64_offset:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 42
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: s_mov_b32 m0, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1] offset:32
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: lds_atomic_inc_ret_i64_offset:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 42
; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll
index c5d213a1ade8b..231bf8e610d33 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.f16.ll
@@ -195,25 +195,25 @@ entry:
define amdgpu_kernel void @class_f16_1(
; VI-SDAG-LABEL: class_f16_1:
; VI-SDAG: ; %bb.0: ; %entry
-; VI-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-SDAG-NEXT: s_mov_b32 s3, 0x1100f000
-; VI-SDAG-NEXT: s_mov_b32 s2, -1
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-SDAG-NEXT: s_mov_b32 s7, 0x1100f000
+; VI-SDAG-NEXT: s_mov_b32 s6, -1
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], s4, 1
-; VI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-SDAG-NEXT: s_mov_b32 s4, s0
+; VI-SDAG-NEXT: s_mov_b32 s5, s1
+; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[0:1], s2, 1
+; VI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; VI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-SDAG-NEXT: s_endpgm
;
; VI-GISEL-LABEL: class_f16_1:
; VI-GISEL: ; %bb.0: ; %entry
-; VI-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
-; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_cmp_class_f16_e64 s[4:5], s3, 1
+; VI-GISEL-NEXT: v_cmp_class_f16_e64 s[4:5], s2, 1
; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-GISEL-NEXT: s_cselect_b32 s3, -1, 0
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
; VI-GISEL-NEXT: s_mov_b32 s3, 0x1100f000
; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -230,25 +230,25 @@ entry:
define amdgpu_kernel void @class_f16_64(
; VI-SDAG-LABEL: class_f16_64:
; VI-SDAG: ; %bb.0: ; %entry
-; VI-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-SDAG-NEXT: s_mov_b32 s3, 0x1100f000
-; VI-SDAG-NEXT: s_mov_b32 s2, -1
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-SDAG-NEXT: s_mov_b32 s7, 0x1100f000
+; VI-SDAG-NEXT: s_mov_b32 s6, -1
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[4:5], s4, 64
-; VI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-SDAG-NEXT: s_mov_b32 s4, s0
+; VI-SDAG-NEXT: s_mov_b32 s5, s1
+; VI-SDAG-NEXT: v_cmp_class_f16_e64 s[0:1], s2, 64
+; VI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; VI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-SDAG-NEXT: s_endpgm
;
; VI-GISEL-LABEL: class_f16_64:
; VI-GISEL: ; %bb.0: ; %entry
-; VI-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
-; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-GISEL-NEXT: s_mov_b32 s2, -1
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_cmp_class_f16_e64 s[4:5], s3, 64
+; VI-GISEL-NEXT: v_cmp_class_f16_e64 s[4:5], s2, 64
; VI-GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; VI-GISEL-NEXT: s_cselect_b32 s3, -1, 0
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
; VI-GISEL-NEXT: s_mov_b32 s3, 0x1100f000
; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -265,27 +265,27 @@ entry:
define amdgpu_kernel void @class_f16_full_mask(
; VI-SDAG-LABEL: class_f16_full_mask:
; VI-SDAG: ; %bb.0: ; %entry
-; VI-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x3ff
-; VI-SDAG-NEXT: s_mov_b32 s3, 0x1100f000
-; VI-SDAG-NEXT: s_mov_b32 s2, -1
+; VI-SDAG-NEXT: s_mov_b32 s7, 0x1100f000
+; VI-SDAG-NEXT: s_mov_b32 s6, -1
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_class_f16_e32 vcc, s4, v0
+; VI-SDAG-NEXT: v_cmp_class_f16_e32 vcc, s2, v0
+; VI-SDAG-NEXT: s_mov_b32 s4, s0
+; VI-SDAG-NEXT: s_mov_b32 s5, s1
; VI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-SDAG-NEXT: s_endpgm
;
; VI-GISEL-LABEL: class_f16_full_mask:
; VI-GISEL: ; %bb.0: ; %entry
-; VI-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
-; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x3ff
-; VI-GISEL-NEXT: s_mov_b32 s2, -1
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_cmp_class_f16_e32 vcc, s3, v0
+; VI-GISEL-NEXT: v_cmp_class_f16_e32 vcc, s2, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s3, -1, 0
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
; VI-GISEL-NEXT: s_mov_b32 s3, 0x1100f000
; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -302,27 +302,27 @@ entry:
define amdgpu_kernel void @class_f16_nine_bit_mask(
; VI-SDAG-LABEL: class_f16_nine_bit_mask:
; VI-SDAG: ; %bb.0: ; %entry
-; VI-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x1ff
-; VI-SDAG-NEXT: s_mov_b32 s3, 0x1100f000
-; VI-SDAG-NEXT: s_mov_b32 s2, -1
+; VI-SDAG-NEXT: s_mov_b32 s7, 0x1100f000
+; VI-SDAG-NEXT: s_mov_b32 s6, -1
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_class_f16_e32 vcc, s4, v0
+; VI-SDAG-NEXT: v_cmp_class_f16_e32 vcc, s2, v0
+; VI-SDAG-NEXT: s_mov_b32 s4, s0
+; VI-SDAG-NEXT: s_mov_b32 s5, s1
; VI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; VI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-SDAG-NEXT: s_endpgm
;
; VI-GISEL-LABEL: class_f16_nine_bit_mask:
; VI-GISEL: ; %bb.0: ; %entry
-; VI-GISEL-NEXT: s_load_dword s3, s[8:9], 0x8
-; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-GISEL-NEXT: v_mov_b32_e32 v0, 0x1ff
-; VI-GISEL-NEXT: s_mov_b32 s2, -1
; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_cmp_class_f16_e32 vcc, s3, v0
+; VI-GISEL-NEXT: v_cmp_class_f16_e32 vcc, s2, v0
; VI-GISEL-NEXT: s_cmp_lg_u64 vcc, 0
; VI-GISEL-NEXT: s_cselect_b32 s3, -1, 0
+; VI-GISEL-NEXT: s_mov_b32 s2, -1
; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
; VI-GISEL-NEXT: s_mov_b32 s3, 0x1100f000
; VI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.ll
index cecb187028924..0f35c333c2908 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.class.ll
@@ -159,14 +159,15 @@ define amdgpu_kernel void @test_class_fneg_fabs_f32(ptr addrspace(1) %out, [8 x
define amdgpu_kernel void @test_class_1_f32(ptr addrspace(1) %out, float %a) #0 {
; SI-SDAG-LABEL: test_class_1_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cmp_class_f32_e64 s[4:5], s6, 1
-; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_cmp_class_f32_e64 s[0:1], s2, 1
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: test_class_1_f32:
@@ -191,14 +192,15 @@ define amdgpu_kernel void @test_class_1_f32(ptr addrspace(1) %out, float %a) #0
define amdgpu_kernel void @test_class_64_f32(ptr addrspace(1) %out, float %a) #0 {
; SI-SDAG-LABEL: test_class_64_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cmp_class_f32_e64 s[4:5], s6, 64
-; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_cmp_class_f32_e64 s[0:1], s2, 64
+; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: test_class_64_f32:
@@ -224,15 +226,16 @@ define amdgpu_kernel void @test_class_64_f32(ptr addrspace(1) %out, float %a) #0
define amdgpu_kernel void @test_class_full_mask_f32(ptr addrspace(1) %out, float %a) #0 {
; SI-SDAG-LABEL: test_class_full_mask_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x3ff
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cmp_class_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: test_class_full_mask_f32:
@@ -258,15 +261,16 @@ define amdgpu_kernel void @test_class_full_mask_f32(ptr addrspace(1) %out, float
define amdgpu_kernel void @test_class_9bit_mask_f32(ptr addrspace(1) %out, float %a) #0 {
; SI-SDAG-LABEL: test_class_9bit_mask_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-SDAG-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x1ff
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cmp_class_f32_e32 vcc, s4, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: v_cmp_class_f32_e32 vcc, s2, v0
; SI-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: test_class_9bit_mask_f32:
@@ -1189,11 +1193,11 @@ define amdgpu_kernel void @test_no_fold_or_class_f32_0(ptr addrspace(1) %out, pt
define amdgpu_kernel void @test_class_0_f32(ptr addrspace(1) %out, float %a) #0 {
; SI-SDAG-LABEL: test_class_0_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
; SI-SDAG-NEXT: s_mov_b32 s2, -1
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.i16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.i16.ll
index 9d2c6fedbec9c..3c7eb124cf7b7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.i16.ll
@@ -33,16 +33,27 @@ define amdgpu_kernel void @s_cvt_pk_i16_i32(ptr addrspace(1) %out, i32 %x, i32 %
; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
-; VI-LABEL: s_cvt_pk_i16_i32:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_cvt_pk_i16_i32 v2, s2, v0
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; VI-SDAG-LABEL: s_cvt_pk_i16_i32:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s3
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_cvt_pk_i16_i32 v2, s2, v2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: s_cvt_pk_i16_i32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: v_cvt_pk_i16_i32 v2, s2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: s_cvt_pk_i16_i32:
; GFX9-SDAG: ; %bb.0:
@@ -72,12 +83,12 @@ define amdgpu_kernel void @s_cvt_pk_i16_i32(ptr addrspace(1) %out, i32 %x, i32 %
define amdgpu_kernel void @s_cvt_pk_i16_samereg_i32(ptr addrspace(1) %out, i32 %x) #0 {
; SI-SDAG-LABEL: s_cvt_pk_i16_samereg_i32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s4, s2
; SI-SDAG-NEXT: s_mov_b32 s2, -1
-; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cvt_pk_i16_i32_e64 v0, s6, s6
+; SI-SDAG-NEXT: v_cvt_pk_i16_i32_e64 v0, s4, s4
; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-SDAG-NEXT: s_endpgm
;
@@ -92,21 +103,30 @@ define amdgpu_kernel void @s_cvt_pk_i16_samereg_i32(ptr addrspace(1) %out, i32 %
; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
-; VI-LABEL: s_cvt_pk_i16_samereg_i32:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cvt_pk_i16_i32 v2, s2, s2
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; VI-SDAG-LABEL: s_cvt_pk_i16_samereg_i32:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_cvt_pk_i16_i32 v2, s2, s2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: s_cvt_pk_i16_samereg_i32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_pk_i16_i32 v2, s2, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: s_cvt_pk_i16_samereg_i32:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_cvt_pk_i16_i32 v1, s2, s2
@@ -429,3 +449,5 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; VI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.u16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.u16.ll
index a098a1161e747..31c64a8d98ffe 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.u16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pk.u16.ll
@@ -33,16 +33,27 @@ define amdgpu_kernel void @s_cvt_pk_u16_u32(ptr addrspace(1) %out, i32 %x, i32 %
; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
-; VI-LABEL: s_cvt_pk_u16_u32:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_cvt_pk_u16_u32 v2, s2, v0
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; VI-SDAG-LABEL: s_cvt_pk_u16_u32:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s3
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_cvt_pk_u16_u32 v2, s2, v2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: s_cvt_pk_u16_u32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: v_cvt_pk_u16_u32 v2, s2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: s_cvt_pk_u16_u32:
; GFX9-SDAG: ; %bb.0:
@@ -72,12 +83,12 @@ define amdgpu_kernel void @s_cvt_pk_u16_u32(ptr addrspace(1) %out, i32 %x, i32 %
define amdgpu_kernel void @s_cvt_pk_u16_samereg_i32(ptr addrspace(1) %out, i32 %x) #0 {
; SI-SDAG-LABEL: s_cvt_pk_u16_samereg_i32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s4, s2
; SI-SDAG-NEXT: s_mov_b32 s2, -1
-; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cvt_pk_u16_u32_e64 v0, s6, s6
+; SI-SDAG-NEXT: v_cvt_pk_u16_u32_e64 v0, s4, s4
; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-SDAG-NEXT: s_endpgm
;
@@ -92,21 +103,30 @@ define amdgpu_kernel void @s_cvt_pk_u16_samereg_i32(ptr addrspace(1) %out, i32 %
; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
-; VI-LABEL: s_cvt_pk_u16_samereg_i32:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cvt_pk_u16_u32 v2, s2, s2
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; VI-SDAG-LABEL: s_cvt_pk_u16_samereg_i32:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_cvt_pk_u16_u32 v2, s2, s2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: s_cvt_pk_u16_samereg_i32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_pk_u16_u32 v2, s2, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: s_cvt_pk_u16_samereg_i32:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_cvt_pk_u16_u32 v1, s2, s2
@@ -429,3 +449,5 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; VI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pknorm.i16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pknorm.i16.ll
index e991ad19005de..59cc4afd5ecb6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pknorm.i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pknorm.i16.ll
@@ -22,10 +22,10 @@ define amdgpu_kernel void @s_cvt_pknorm_i16_f32(ptr addrspace(1) %out, float %x,
; VI-LABEL: s_cvt_pknorm_i16_f32:
; VI: ; %bb.0:
; VI: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI: v_mov_b32_e32 v0, s3
-; VI: v_cvt_pknorm_i16_f32 v2, s2, v0
+; VI: v_mov_b32_e32 v2, s3
; VI: v_mov_b32_e32 v0, s0
; VI: v_mov_b32_e32 v1, s1
+; VI: v_cvt_pknorm_i16_f32 v2, s2, v2
; VI: s_endpgm
;
; GFX9-SDAG-LABEL: s_cvt_pknorm_i16_f32:
@@ -52,26 +52,24 @@ define amdgpu_kernel void @s_cvt_pknorm_i16_f32(ptr addrspace(1) %out, float %x,
define amdgpu_kernel void @s_cvt_pknorm_i16_samereg_f32(ptr addrspace(1) %out, float %x) #0 {
; SI-LABEL: s_cvt_pknorm_i16_samereg_f32:
; SI: ; %bb.0:
-; SI: s_load_dword s6, s[4:5], 0xb
-; SI: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI: s_mov_b32 s3, 0xf000
+; SI: s_mov_b32 s4, s2
; SI: s_mov_b32 s2, -1
-; SI: v_cvt_pknorm_i16_f32_e64 v0, s6, s6
+; SI: v_cvt_pknorm_i16_f32_e64 v0, s4, s4
; SI: s_endpgm
;
; VI-LABEL: s_cvt_pknorm_i16_samereg_f32:
; VI: ; %bb.0:
-; VI: s_load_dword s2, s[4:5], 0x2c
-; VI: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI: v_cvt_pknorm_i16_f32 v2, s2, s2
+; VI: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI: v_mov_b32_e32 v0, s0
; VI: v_mov_b32_e32 v1, s1
+; VI: v_cvt_pknorm_i16_f32 v2, s2, s2
; VI: s_endpgm
;
; GFX9-SDAG-LABEL: s_cvt_pknorm_i16_samereg_f32:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG: s_load_dword s2, s[4:5], 0x2c
-; GFX9-SDAG: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-SDAG: v_mov_b32_e32 v0, 0
; GFX9-SDAG: v_cvt_pknorm_i16_f32 v1, s2, s2
; GFX9-SDAG: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pknorm.u16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pknorm.u16.ll
index 5b81795f456e6..7dbcf3372c404 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pknorm.u16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pknorm.u16.ll
@@ -22,10 +22,10 @@ define amdgpu_kernel void @s_cvt_pknorm_u16_f32(ptr addrspace(1) %out, float %x,
; VI-LABEL: s_cvt_pknorm_u16_f32:
; VI: ; %bb.0:
; VI: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI: v_mov_b32_e32 v0, s3
-; VI: v_cvt_pknorm_u16_f32 v2, s2, v0
+; VI: v_mov_b32_e32 v2, s3
; VI: v_mov_b32_e32 v0, s0
; VI: v_mov_b32_e32 v1, s1
+; VI: v_cvt_pknorm_u16_f32 v2, s2, v2
; VI: s_endpgm
;
; GFX9-SDAG-LABEL: s_cvt_pknorm_u16_f32:
@@ -52,26 +52,24 @@ define amdgpu_kernel void @s_cvt_pknorm_u16_f32(ptr addrspace(1) %out, float %x,
define amdgpu_kernel void @s_cvt_pknorm_u16_samereg_f32(ptr addrspace(1) %out, float %x) #0 {
; SI-LABEL: s_cvt_pknorm_u16_samereg_f32:
; SI: ; %bb.0:
-; SI: s_load_dword s6, s[4:5], 0xb
-; SI: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI: s_mov_b32 s3, 0xf000
+; SI: s_mov_b32 s4, s2
; SI: s_mov_b32 s2, -1
-; SI: v_cvt_pknorm_u16_f32_e64 v0, s6, s6
+; SI: v_cvt_pknorm_u16_f32_e64 v0, s4, s4
; SI: s_endpgm
;
; VI-LABEL: s_cvt_pknorm_u16_samereg_f32:
; VI: ; %bb.0:
-; VI: s_load_dword s2, s[4:5], 0x2c
-; VI: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI: v_cvt_pknorm_u16_f32 v2, s2, s2
+; VI: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI: v_mov_b32_e32 v0, s0
; VI: v_mov_b32_e32 v1, s1
+; VI: v_cvt_pknorm_u16_f32 v2, s2, s2
; VI: s_endpgm
;
; GFX9-SDAG-LABEL: s_cvt_pknorm_u16_samereg_f32:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG: s_load_dword s2, s[4:5], 0x2c
-; GFX9-SDAG: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-SDAG: v_mov_b32_e32 v0, 0
; GFX9-SDAG: v_cvt_pknorm_u16_f32 v1, s2, s2
; GFX9-SDAG: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
index 2f84a0c73f9dc..4a996c402a1db 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.cvt.pkrtz.ll
@@ -35,16 +35,27 @@ define amdgpu_kernel void @s_cvt_pkrtz_v2f16_f32(ptr addrspace(1) %out, float %x
; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
-; VI-LABEL: s_cvt_pkrtz_v2f16_f32:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s3
-; VI-NEXT: v_cvt_pkrtz_f16_f32 v2, s2, v0
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; VI-SDAG-LABEL: s_cvt_pkrtz_v2f16_f32:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s3
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_cvt_pkrtz_f16_f32 v2, s2, v2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: s_cvt_pkrtz_v2f16_f32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT: v_cvt_pkrtz_f16_f32 v2, s2, v0
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: s_cvt_pkrtz_v2f16_f32:
; GFX9-SDAG: ; %bb.0:
@@ -109,12 +120,12 @@ define amdgpu_kernel void @s_cvt_pkrtz_v2f16_f32(ptr addrspace(1) %out, float %x
define amdgpu_kernel void @s_cvt_pkrtz_samereg_v2f16_f32(ptr addrspace(1) %out, float %x) #0 {
; SI-SDAG-LABEL: s_cvt_pkrtz_samereg_v2f16_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s4, s2
; SI-SDAG-NEXT: s_mov_b32 s2, -1
-; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cvt_pkrtz_f16_f32_e64 v0, s6, s6
+; SI-SDAG-NEXT: v_cvt_pkrtz_f16_f32_e64 v0, s4, s4
; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-SDAG-NEXT: s_endpgm
;
@@ -129,21 +140,30 @@ define amdgpu_kernel void @s_cvt_pkrtz_samereg_v2f16_f32(ptr addrspace(1) %out,
; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-GISEL-NEXT: s_endpgm
;
-; VI-LABEL: s_cvt_pkrtz_samereg_v2f16_f32:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cvt_pkrtz_f16_f32 v2, s2, s2
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
+; VI-SDAG-LABEL: s_cvt_pkrtz_samereg_v2f16_f32:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_cvt_pkrtz_f16_f32 v2, s2, s2
+; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
+; VI-SDAG-NEXT: s_endpgm
+;
+; VI-GISEL-LABEL: s_cvt_pkrtz_samereg_v2f16_f32:
+; VI-GISEL: ; %bb.0:
+; VI-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT: v_cvt_pkrtz_f16_f32 v2, s2, s2
+; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
+; VI-GISEL-NEXT: s_endpgm
;
; GFX9-SDAG-LABEL: s_cvt_pkrtz_samereg_v2f16_f32:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-SDAG-NEXT: v_cvt_pkrtz_f16_f32 v1, s2, s2
@@ -162,9 +182,7 @@ define amdgpu_kernel void @s_cvt_pkrtz_samereg_v2f16_f32(ptr addrspace(1) %out,
;
; GFX10-SDAG-LABEL: s_cvt_pkrtz_samereg_v2f16_f32:
; GFX10-SDAG: ; %bb.0:
-; GFX10-SDAG-NEXT: s_clause 0x1
-; GFX10-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX10-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-SDAG-NEXT: v_cvt_pkrtz_f16_f32_e64 v1, s2, s2
@@ -184,9 +202,7 @@ define amdgpu_kernel void @s_cvt_pkrtz_samereg_v2f16_f32(ptr addrspace(1) %out,
;
; GFX11-SDAG-LABEL: s_cvt_pkrtz_samereg_v2f16_f32:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_cvt_pk_rtz_f16_f32_e64 v1, s2, s2
@@ -1473,3 +1489,5 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; VI: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll
index 18afb7c677207..0ec989f90a250 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.div.fmas.ll
@@ -57,19 +57,21 @@ define amdgpu_kernel void @test_div_fmas_f32_inline_imm_0(ptr addrspace(1) %out,
define amdgpu_kernel void @test_div_fmas_f32_inline_imm_1(ptr addrspace(1) %out, float %a, float %b, float %c, [8 x i32], i1 %d) nounwind {
; GCN-LABEL: test_div_fmas_f32_inline_imm_1:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x16
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dword s4, s[4:5], 0xd
-; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_load_dword s6, s[4:5], 0x16
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_bitcmp1_b32 s2, 0
+; GCN-NEXT: s_load_dword s3, s[4:5], 0xd
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_bitcmp1_b32 s6, 0
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s6
-; GCN-NEXT: v_mov_b32_e32 v1, s4
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v1, s3
; GCN-NEXT: v_div_fmas_f32 v0, v0, 1.0, v1
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%result = call float @llvm.amdgcn.div.fmas.f32(float %a, float 1.0, float %c, i1 %d) nounwind readnone
store float %result, ptr addrspace(1) %out, align 4
@@ -127,16 +129,18 @@ define amdgpu_kernel void @test_div_fmas_f64(ptr addrspace(1) %out, double %a, d
define amdgpu_kernel void @test_div_fmas_f32_cond_to_vcc(ptr addrspace(1) %out, float %a, float %b, float %c, i32 %i) nounwind {
; GCN-LABEL: test_div_fmas_f32_cond_to_vcc:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s3, 0
+; GCN-NEXT: s_cmp_eq_u32 s9, 0
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v2, s8
; GCN-NEXT: v_div_fmas_f32 v0, v0, v1, v2
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
@@ -149,14 +153,16 @@ define amdgpu_kernel void @test_div_fmas_f32_cond_to_vcc(ptr addrspace(1) %out,
define amdgpu_kernel void @test_div_fmas_f32_imm_false_cond_to_vcc(ptr addrspace(1) %out, float %a, float %b, float %c) nounwind {
; GCN-LABEL: test_div_fmas_f32_imm_false_cond_to_vcc:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_load_dword s8, s[4:5], 0xd
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v2, s8
; GCN-NEXT: s_mov_b64 vcc, 0
; GCN-NEXT: v_div_fmas_f32 v0, v0, v1, v2
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
@@ -169,14 +175,16 @@ define amdgpu_kernel void @test_div_fmas_f32_imm_false_cond_to_vcc(ptr addrspace
define amdgpu_kernel void @test_div_fmas_f32_imm_true_cond_to_vcc(ptr addrspace(1) %out, float %a, float %b, float %c) nounwind {
; GCN-LABEL: test_div_fmas_f32_imm_true_cond_to_vcc:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_load_dword s8, s[4:5], 0xd
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s1
-; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: v_mov_b32_e32 v2, s8
; GCN-NEXT: s_mov_b64 vcc, -1
; GCN-NEXT: v_div_fmas_f32 v0, v0, v1, v2
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.swizzle.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.swizzle.ll
index 732e501d580ec..71ec9bac23e0a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.swizzle.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.swizzle.ll
@@ -5,31 +5,6 @@
; RUN: llc -global-isel=1 -new-reg-bank-select -mtriple=amdgcn -mcpu=fiji < %s | FileCheck -check-prefixes=CHECK,GFX8 %s
define amdgpu_kernel void @ds_swizzle(ptr addrspace(1) %out, i32 %src) nounwind {
-; GFX7-LABEL: ds_swizzle:
-; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dword s2, s[4:5], 0xb
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_mov_b32_e32 v0, s2
-; GFX7-NEXT: ds_swizzle_b32 v0, v0 offset:swizzle(BITMASK_PERM,"00p11")
-; GFX7-NEXT: s_mov_b32 s2, -1
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; GFX7-NEXT: s_endpgm
-;
-; GFX8-LABEL: ds_swizzle:
-; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: ds_swizzle_b32 v2, v0 offset:swizzle(BITMASK_PERM,"00p11")
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: flat_store_dword v[0:1], v2
-; GFX8-NEXT: s_endpgm
%swizzle = call i32 @llvm.amdgcn.ds.swizzle(i32 %src, i32 100) #0
store i32 %swizzle, ptr addrspace(1) %out, align 4
ret void
@@ -61,3 +36,6 @@ define amdgpu_ps i32 @ds_swizzle_v(i32 %src) {
declare i32 @llvm.amdgcn.ds.swizzle(i32, i32) #0
attributes #0 = { nounwind readnone convergent }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX7: {{.*}}
+; GFX8: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
index 87a9ba30490a0..296d26785b8c0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w32.ll
@@ -139,9 +139,7 @@ define amdgpu_kernel void @v_fcmp_f32(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_oeq(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_oeq:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_eq_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -151,9 +149,7 @@ define amdgpu_kernel void @v_fcmp_f32_oeq(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_oeq:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_eq_f32_e64 s2, 0x42c80000, s2
@@ -193,9 +189,7 @@ define amdgpu_kernel void @v_fcmp_f32_oeq(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_one(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_one:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_neq_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -205,9 +199,7 @@ define amdgpu_kernel void @v_fcmp_f32_one(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_one:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_neq_f32_e64 s2, 0x42c80000, s2
@@ -247,9 +239,7 @@ define amdgpu_kernel void @v_fcmp_f32_one(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_ogt(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_ogt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_lt_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -259,9 +249,7 @@ define amdgpu_kernel void @v_fcmp_f32_ogt(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_ogt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_lt_f32_e64 s2, 0x42c80000, s2
@@ -301,9 +289,7 @@ define amdgpu_kernel void @v_fcmp_f32_ogt(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_oge(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_oge:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_le_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -313,9 +299,7 @@ define amdgpu_kernel void @v_fcmp_f32_oge(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_oge:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_le_f32_e64 s2, 0x42c80000, s2
@@ -355,9 +339,7 @@ define amdgpu_kernel void @v_fcmp_f32_oge(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_olt(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_olt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_gt_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -367,9 +349,7 @@ define amdgpu_kernel void @v_fcmp_f32_olt(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_olt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_gt_f32_e64 s2, 0x42c80000, s2
@@ -409,9 +389,7 @@ define amdgpu_kernel void @v_fcmp_f32_olt(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_ole(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_ole:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_ge_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -421,9 +399,7 @@ define amdgpu_kernel void @v_fcmp_f32_ole(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_ole:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_ge_f32_e64 s2, 0x42c80000, s2
@@ -463,9 +439,7 @@ define amdgpu_kernel void @v_fcmp_f32_ole(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_o(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_o:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_o_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -475,9 +449,7 @@ define amdgpu_kernel void @v_fcmp_f32_o(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_o:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_o_f32_e64 s2, 0x42c80000, s2
@@ -517,9 +489,7 @@ define amdgpu_kernel void @v_fcmp_f32_o(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_uo(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_uo:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_u_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -529,9 +499,7 @@ define amdgpu_kernel void @v_fcmp_f32_uo(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_uo:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_u_f32_e64 s2, 0x42c80000, s2
@@ -571,9 +539,7 @@ define amdgpu_kernel void @v_fcmp_f32_uo(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_ueq(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_ueq:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_nlg_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -583,9 +549,7 @@ define amdgpu_kernel void @v_fcmp_f32_ueq(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_ueq:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_nlg_f32_e64 s2, 0x42c80000, s2
@@ -625,9 +589,7 @@ define amdgpu_kernel void @v_fcmp_f32_ueq(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_une(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_une:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_neq_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -637,9 +599,7 @@ define amdgpu_kernel void @v_fcmp_f32_une(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_une:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_neq_f32_e64 s2, 0x42c80000, s2
@@ -679,9 +639,7 @@ define amdgpu_kernel void @v_fcmp_f32_une(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_ugt(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_ugt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_nge_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -691,9 +649,7 @@ define amdgpu_kernel void @v_fcmp_f32_ugt(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_ugt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_nge_f32_e64 s2, 0x42c80000, s2
@@ -733,9 +689,7 @@ define amdgpu_kernel void @v_fcmp_f32_ugt(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_uge(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_uge:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_ngt_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -745,9 +699,7 @@ define amdgpu_kernel void @v_fcmp_f32_uge(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_uge:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_ngt_f32_e64 s2, 0x42c80000, s2
@@ -787,9 +739,7 @@ define amdgpu_kernel void @v_fcmp_f32_uge(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_ult(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_ult:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_nle_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -799,9 +749,7 @@ define amdgpu_kernel void @v_fcmp_f32_ult(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_ult:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_nle_f32_e64 s2, 0x42c80000, s2
@@ -841,9 +789,7 @@ define amdgpu_kernel void @v_fcmp_f32_ult(ptr addrspace(1) %out, float %src) {
define amdgpu_kernel void @v_fcmp_f32_ule(ptr addrspace(1) %out, float %src) {
; SDAG-GFX11-LABEL: v_fcmp_f32_ule:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_nlt_f32_e64 s2, 0x42c80000, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -853,9 +799,7 @@ define amdgpu_kernel void @v_fcmp_f32_ule(ptr addrspace(1) %out, float %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f32_ule:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_nlt_f32_e64 s2, 0x42c80000, s2
@@ -1540,9 +1484,7 @@ define amdgpu_kernel void @v_fcmp_f64_ule(ptr addrspace(1) %out, double %src) {
define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half %src, half %a) {
; SDAG-GFX11-LABEL: v_fcmp_f16_oeq_with_fabs:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: s_lshr_b32 s3, s2, 16
; SDAG-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1553,9 +1495,7 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half
;
; SDAG-GFX10-LABEL: v_fcmp_f16_oeq_with_fabs:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: s_lshr_b32 s3, s2, 16
@@ -1600,9 +1540,7 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half
define amdgpu_kernel void @v_fcmp_f16_oeq_both_operands_with_fabs(ptr addrspace(1) %out, half %src, half %a) {
; SDAG-GFX11-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: s_lshr_b32 s3, s2, 16
; SDAG-GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -1613,9 +1551,7 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_both_operands_with_fabs(ptr addrspace(
;
; SDAG-GFX10-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: s_lshr_b32 s3, s2, 16
@@ -1690,9 +1626,7 @@ define amdgpu_kernel void @v_fcmp_f16(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_oeq(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_oeq:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_eq_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1702,9 +1636,7 @@ define amdgpu_kernel void @v_fcmp_f16_oeq(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_oeq:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_eq_f16_e64 s2, 0x5640, s2
@@ -1745,9 +1677,7 @@ define amdgpu_kernel void @v_fcmp_f16_oeq(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_one(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_one:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_neq_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1757,9 +1687,7 @@ define amdgpu_kernel void @v_fcmp_f16_one(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_one:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_neq_f16_e64 s2, 0x5640, s2
@@ -1800,9 +1728,7 @@ define amdgpu_kernel void @v_fcmp_f16_one(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_ogt(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_ogt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_lt_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1812,9 +1738,7 @@ define amdgpu_kernel void @v_fcmp_f16_ogt(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_ogt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_lt_f16_e64 s2, 0x5640, s2
@@ -1855,9 +1779,7 @@ define amdgpu_kernel void @v_fcmp_f16_ogt(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_oge(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_oge:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_le_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1867,9 +1789,7 @@ define amdgpu_kernel void @v_fcmp_f16_oge(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_oge:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_le_f16_e64 s2, 0x5640, s2
@@ -1910,9 +1830,7 @@ define amdgpu_kernel void @v_fcmp_f16_oge(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_olt(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_olt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_gt_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1922,9 +1840,7 @@ define amdgpu_kernel void @v_fcmp_f16_olt(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_olt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_gt_f16_e64 s2, 0x5640, s2
@@ -1965,9 +1881,7 @@ define amdgpu_kernel void @v_fcmp_f16_olt(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_ole(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_ole:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_ge_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1977,9 +1891,7 @@ define amdgpu_kernel void @v_fcmp_f16_ole(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_ole:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_ge_f16_e64 s2, 0x5640, s2
@@ -2020,9 +1932,7 @@ define amdgpu_kernel void @v_fcmp_f16_ole(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_ueq(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_ueq:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_nlg_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2032,9 +1942,7 @@ define amdgpu_kernel void @v_fcmp_f16_ueq(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_ueq:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_nlg_f16_e64 s2, 0x5640, s2
@@ -2075,9 +1983,7 @@ define amdgpu_kernel void @v_fcmp_f16_ueq(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_une(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_une:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_neq_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2087,9 +1993,7 @@ define amdgpu_kernel void @v_fcmp_f16_une(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_une:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_neq_f16_e64 s2, 0x5640, s2
@@ -2130,9 +2034,7 @@ define amdgpu_kernel void @v_fcmp_f16_une(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_ugt(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_ugt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_nge_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2142,9 +2044,7 @@ define amdgpu_kernel void @v_fcmp_f16_ugt(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_ugt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_nge_f16_e64 s2, 0x5640, s2
@@ -2185,9 +2085,7 @@ define amdgpu_kernel void @v_fcmp_f16_ugt(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_uge(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_uge:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_ngt_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2197,9 +2095,7 @@ define amdgpu_kernel void @v_fcmp_f16_uge(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_uge:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_ngt_f16_e64 s2, 0x5640, s2
@@ -2240,9 +2136,7 @@ define amdgpu_kernel void @v_fcmp_f16_uge(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_ult(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_ult:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_nle_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2252,9 +2146,7 @@ define amdgpu_kernel void @v_fcmp_f16_ult(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_ult:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_nle_f16_e64 s2, 0x5640, s2
@@ -2294,9 +2186,7 @@ define amdgpu_kernel void @v_fcmp_f16_ult(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_o(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_o:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_o_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2306,9 +2196,7 @@ define amdgpu_kernel void @v_fcmp_f16_o(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_o:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_o_f16_e64 s2, 0x5640, s2
@@ -2348,9 +2236,7 @@ define amdgpu_kernel void @v_fcmp_f16_o(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_uo(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_uo:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_u_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2360,9 +2246,7 @@ define amdgpu_kernel void @v_fcmp_f16_uo(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_uo:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_u_f16_e64 s2, 0x5640, s2
@@ -2402,9 +2286,7 @@ define amdgpu_kernel void @v_fcmp_f16_uo(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_ule(ptr addrspace(1) %out, half %src) {
; SDAG-GFX11-LABEL: v_fcmp_f16_ule:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_nlt_f16_e64 s2, 0x5640, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -2414,9 +2296,7 @@ define amdgpu_kernel void @v_fcmp_f16_ule(ptr addrspace(1) %out, half %src) {
;
; SDAG-GFX10-LABEL: v_fcmp_f16_ule:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_nlt_f16_e64 s2, 0x5640, s2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
index 9e4824694e76a..37943096ac043 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
@@ -42,12 +42,12 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_with_fabs(ptr addrspace(1) %out, float
; VI-SDAG: ; %bb.0:
; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, s3
-; VI-SDAG-NEXT: v_cmp_eq_f32_e64 s[2:3], s2, |v0|
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s3
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_eq_f32_e64 s[0:1], s2, |v2|
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -98,12 +98,12 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_both_operands_with_fabs(ptr addrspace(
; VI-SDAG: ; %bb.0:
; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, s3
-; VI-SDAG-NEXT: v_cmp_eq_f32_e64 s[2:3], |s2|, |v0|
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s3
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_eq_f32_e64 s[0:1], |s2|, |v2|
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -169,44 +169,67 @@ define amdgpu_kernel void @v_fcmp_f32(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_oeq(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_oeq:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_eq_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_oeq:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_eq_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_oeq:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_oeq:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_eq_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_oeq:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_oeq:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_oeq:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_eq_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_eq_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -229,44 +252,67 @@ define amdgpu_kernel void @v_fcmp_f32_oeq(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_one(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_one:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_neq_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_one:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_neq_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_one:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_neq_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_one:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_neq_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_one:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_neq_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_one:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_neq_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_one:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_neq_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_neq_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -289,44 +335,67 @@ define amdgpu_kernel void @v_fcmp_f32_one(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_ogt(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_ogt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_lt_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_ogt:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_lt_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_ogt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_gt_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_ogt:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_ogt:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_gt_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_ogt:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_gt_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_ogt:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_gt_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_gt_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -349,44 +418,67 @@ define amdgpu_kernel void @v_fcmp_f32_ogt(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_oge(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_oge:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_le_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_oge:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_le_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_oge:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_oge:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_le_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_oge:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_ge_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_oge:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_ge_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_oge:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_ge_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_ge_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -409,44 +501,67 @@ define amdgpu_kernel void @v_fcmp_f32_oge(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_olt(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_olt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_gt_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_olt:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_gt_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_olt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_lt_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_olt:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_gt_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_olt:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_lt_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_olt:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_lt_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_olt:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -469,44 +584,67 @@ define amdgpu_kernel void @v_fcmp_f32_olt(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_ole(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_ole:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ge_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_ole:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_ge_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_ole:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_le_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_ole:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ge_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_ole:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_le_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_ole:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_le_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_ole:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_le_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_le_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -529,44 +667,67 @@ define amdgpu_kernel void @v_fcmp_f32_ole(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_o(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_o:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_o_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_o:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_o_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_o:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_o_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_o:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_o_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_o:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_o_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_o:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_o_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_o:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_o_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_o_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -589,44 +750,67 @@ define amdgpu_kernel void @v_fcmp_f32_o(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_uo(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_uo:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_u_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_uo:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_u_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_uo:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_u_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_uo:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_u_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
;
-; VI-SDAG-LABEL: v_fcmp_f32_uo:
-; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_u_f32_e64 s[2:3], s2, v0
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; GFX9-SDAG-LABEL: v_fcmp_f32_uo:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_u_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_uo:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_u_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f32_uo:
+; VI-SDAG: ; %bb.0:
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
+; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_cmp_u_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -649,44 +833,67 @@ define amdgpu_kernel void @v_fcmp_f32_uo(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_ueq(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_ueq:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_nlg_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_ueq:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_nlg_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_ueq:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_nlg_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_ueq:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_nlg_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_ueq:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_nlg_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_ueq:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_nlg_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_ueq:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_nlg_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_nlg_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -709,44 +916,67 @@ define amdgpu_kernel void @v_fcmp_f32_ueq(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_une(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_une:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_neq_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_une:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_neq_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_une:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_neq_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_une:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_neq_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_une:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_neq_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_une:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_neq_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_une:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_neq_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_neq_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -769,44 +999,67 @@ define amdgpu_kernel void @v_fcmp_f32_une(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_ugt(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_ugt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_nge_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_ugt:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_nge_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_ugt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_nle_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_ugt:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_nge_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_ugt:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_nle_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_ugt:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_nle_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_ugt:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_nle_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_nle_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -829,44 +1082,67 @@ define amdgpu_kernel void @v_fcmp_f32_ugt(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_uge(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_uge:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ngt_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_uge:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_ngt_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_uge:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_nlt_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_uge:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ngt_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_uge:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_nlt_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_uge:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_nlt_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_uge:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_nlt_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_nlt_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -889,44 +1165,67 @@ define amdgpu_kernel void @v_fcmp_f32_uge(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_ult(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_ult:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_nle_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_ult:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_nle_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_ult:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_nge_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_ult:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_nle_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_ult:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_nge_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_ult:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_nge_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_ult:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_nge_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_nge_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -949,44 +1248,67 @@ define amdgpu_kernel void @v_fcmp_f32_ult(ptr addrspace(1) %out, float %src) {
}
define amdgpu_kernel void @v_fcmp_f32_ule(ptr addrspace(1) %out, float %src) {
-; GFX11-LABEL: v_fcmp_f32_ule:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_nlt_f32_e64 s[2:3], 0x42c80000, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32_ule:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_nlt_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f32_ule:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x42c80000
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_ngt_f32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f32_ule:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_nlt_f32_e64 s[2:3], 0x42c80000, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32_ule:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_ngt_f32_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32_ule:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_ngt_f32_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f32_ule:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42c80000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_ngt_f32_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_ngt_f32_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -1822,48 +2144,74 @@ define amdgpu_kernel void @v_fcmp_f64_ule(ptr addrspace(1) %out, double %src) {
define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half %src, half %a) {
-; GFX11-LABEL: v_fcmp_f16_oeq_with_fabs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |s3|
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_oeq_with_fabs:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |s3|
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_oeq_with_fabs:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s3, s2, 16
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |v0|
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_oeq_with_fabs:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |s3|
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_oeq_with_fabs:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |v0|
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_oeq_with_fabs:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |v0|
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_oeq_with_fabs:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: s_lshr_b32 s3, s2, 16
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, s3
-; VI-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, |v0|
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_eq_f16_e64 s[0:1], s2, |v2|
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -1889,48 +2237,74 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half
define amdgpu_kernel void @v_fcmp_f16_oeq_both_operands_with_fabs(ptr addrspace(1) %out, half %src, half %a) {
-; GFX11-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshr_b32 s3, s2, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |s3|
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
-;
-; GFX9-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s3, s2, 16
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |s3|
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
+;
+; GFX11-GISEL-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX11-GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |s3|
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: s_lshr_b32 s3, s2, 16
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: s_lshr_b32 s3, s2, 16
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, s3
-; VI-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT: s_lshr_b32 s0, s2, 16
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_eq_f16_e64 s[0:1], |s2|, |v2|
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -1999,44 +2373,67 @@ define amdgpu_kernel void @v_fcmp_f16(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_oeq(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_oeq:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_eq_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_oeq:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_oeq:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_oeq:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_oeq:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_oeq:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_oeq:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_eq_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_eq_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2060,44 +2457,67 @@ define amdgpu_kernel void @v_fcmp_f16_oeq(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_one(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_one:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_neq_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_one:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_neq_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_one:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_neq_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_one:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_neq_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_one:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_neq_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_one:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_neq_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_one:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_neq_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_neq_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2121,44 +2541,67 @@ define amdgpu_kernel void @v_fcmp_f16_one(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_ogt(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_ogt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_lt_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_ogt:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_lt_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_ogt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_gt_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_ogt:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_lt_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_ogt:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_gt_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_ogt:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_gt_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_ogt:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_gt_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_gt_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2182,44 +2625,67 @@ define amdgpu_kernel void @v_fcmp_f16_ogt(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_oge(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_oge:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_le_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_oge:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_le_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_oge:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_ge_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_oge:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_le_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_oge:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_ge_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_oge:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_ge_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_oge:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_ge_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_ge_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2243,44 +2709,67 @@ define amdgpu_kernel void @v_fcmp_f16_oge(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_olt(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_olt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_gt_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_olt:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_gt_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_olt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_lt_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_olt:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_gt_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_olt:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_lt_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_olt:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_lt_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_olt:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_lt_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2304,44 +2793,67 @@ define amdgpu_kernel void @v_fcmp_f16_olt(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_ole(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_ole:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ge_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_ole:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_ge_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_ole:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_le_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_ole:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ge_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_ole:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_le_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_ole:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_le_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_ole:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_le_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_le_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2365,44 +2877,67 @@ define amdgpu_kernel void @v_fcmp_f16_ole(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_ueq(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_ueq:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_nlg_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_ueq:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_nlg_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_ueq:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_nlg_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_ueq:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_nlg_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_ueq:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_nlg_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_ueq:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_nlg_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_ueq:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_nlg_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_nlg_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2426,44 +2961,67 @@ define amdgpu_kernel void @v_fcmp_f16_ueq(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_une(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_une:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_neq_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_une:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_neq_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_une:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_neq_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_une:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_neq_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_une:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_neq_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_une:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_neq_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_une:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_neq_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_neq_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2487,44 +3045,67 @@ define amdgpu_kernel void @v_fcmp_f16_une(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_ugt(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_ugt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_nge_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_ugt:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_nge_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_ugt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_nle_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_ugt:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_nge_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_ugt:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_nle_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_ugt:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_nle_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_ugt:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_nle_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_nle_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2548,44 +3129,67 @@ define amdgpu_kernel void @v_fcmp_f16_ugt(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_uge(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_uge:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ngt_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_uge:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_ngt_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_uge:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_nlt_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_uge:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_ngt_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_uge:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_nlt_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_uge:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_nlt_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_uge:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_nlt_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_nlt_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2609,44 +3213,67 @@ define amdgpu_kernel void @v_fcmp_f16_uge(ptr addrspace(1) %out, half %src) {
define amdgpu_kernel void @v_fcmp_f16_ult(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_ult:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_nle_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_ult:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_nle_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_ult:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_nge_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_ult:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_nle_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_ult:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_nge_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_ult:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_nge_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_ult:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_nge_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_nge_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2669,44 +3296,67 @@ define amdgpu_kernel void @v_fcmp_f16_ult(ptr addrspace(1) %out, half %src) {
}
define amdgpu_kernel void @v_fcmp_f16_o(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_o:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_o_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_o:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_o_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_o:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_o_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_o:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_o_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_o:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_o_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_o:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_o_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_o:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_o_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_o_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2729,44 +3379,67 @@ define amdgpu_kernel void @v_fcmp_f16_o(ptr addrspace(1) %out, half %src) {
}
define amdgpu_kernel void @v_fcmp_f16_uo(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_uo:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_u_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_uo:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_u_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_uo:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_u_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_uo:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_u_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_uo:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_u_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_uo:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_u_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_uo:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_u_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_u_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
@@ -2789,44 +3462,67 @@ define amdgpu_kernel void @v_fcmp_f16_uo(ptr addrspace(1) %out, half %src) {
}
define amdgpu_kernel void @v_fcmp_f16_ule(ptr addrspace(1) %out, half %src) {
-; GFX11-LABEL: v_fcmp_f16_ule:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_nlt_f16_e64 s[2:3], 0x5640, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16_ule:
+; GFX11-SDAG: ; %bb.0:
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-SDAG-NEXT: v_cmp_nlt_f16_e64 s[2:3], 0x5640, s2
+; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-SDAG-NEXT: s_endpgm
;
-; GFX9-LABEL: v_fcmp_f16_ule:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x5640
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_ngt_f16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; GFX11-GISEL-LABEL: v_fcmp_f16_ule:
+; GFX11-GISEL: ; %bb.0:
+; GFX11-GISEL-NEXT: s_clause 0x1
+; GFX11-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX11-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT: v_cmp_nlt_f16_e64 s[2:3], 0x5640, s2
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT: s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16_ule:
+; GFX9-SDAG: ; %bb.0:
+; GFX9-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT: v_cmp_ngt_f16_e64 s[2:3], s2, v0
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-SDAG-NEXT: s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16_ule:
+; GFX9-GISEL: ; %bb.0:
+; GFX9-GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, 0x5640
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT: v_cmp_ngt_f16_e64 s[2:3], s2, v0
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT: s_endpgm
;
; VI-SDAG-LABEL: v_fcmp_f16_ule:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x5640
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_ngt_f16_e64 s[2:3], s2, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s2
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT: v_cmp_ngt_f16_e64 s[0:1], s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.ll
index daabf5c40ea4c..9abb299a4fcab 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.frexp.exp.ll
@@ -13,13 +13,14 @@ declare i32 @llvm.amdgcn.frexp.exp.i32.f64(double) #0
define amdgpu_kernel void @s_test_frexp_exp_f32(ptr addrspace(1) %out, float %src) #1 {
; GFX6-SDAG-LABEL: s_test_frexp_exp_f32:
; GFX6-SDAG: ; %bb.0:
-; GFX6-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX6-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s6, -1
; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s6
-; GFX6-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX6-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s2
+; GFX6-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-SDAG-NEXT: s_endpgm
;
; GFX6-GISEL-LABEL: s_test_frexp_exp_f32:
@@ -35,13 +36,14 @@ define amdgpu_kernel void @s_test_frexp_exp_f32(ptr addrspace(1) %out, float %sr
;
; GFX8-SDAG-LABEL: s_test_frexp_exp_f32:
; GFX8-SDAG: ; %bb.0:
-; GFX8-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s6
-; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX8-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s2
+; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-SDAG-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: s_test_frexp_exp_f32:
@@ -62,13 +64,14 @@ define amdgpu_kernel void @s_test_frexp_exp_f32(ptr addrspace(1) %out, float %sr
define amdgpu_kernel void @s_test_fabs_frexp_exp_f32(ptr addrspace(1) %out, float %src) #1 {
; GFX6-SDAG-LABEL: s_test_fabs_frexp_exp_f32:
; GFX6-SDAG: ; %bb.0:
-; GFX6-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX6-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s6, -1
; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s6
-; GFX6-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX6-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s2
+; GFX6-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-SDAG-NEXT: s_endpgm
;
; GFX6-GISEL-LABEL: s_test_fabs_frexp_exp_f32:
@@ -84,13 +87,14 @@ define amdgpu_kernel void @s_test_fabs_frexp_exp_f32(ptr addrspace(1) %out, floa
;
; GFX8-SDAG-LABEL: s_test_fabs_frexp_exp_f32:
; GFX8-SDAG: ; %bb.0:
-; GFX8-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s6
-; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX8-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s2
+; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-SDAG-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: s_test_fabs_frexp_exp_f32:
@@ -112,13 +116,14 @@ define amdgpu_kernel void @s_test_fabs_frexp_exp_f32(ptr addrspace(1) %out, floa
define amdgpu_kernel void @s_test_fneg_fabs_frexp_exp_f32(ptr addrspace(1) %out, float %src) #1 {
; GFX6-SDAG-LABEL: s_test_fneg_fabs_frexp_exp_f32:
; GFX6-SDAG: ; %bb.0:
-; GFX6-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX6-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s6, -1
; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s6
-; GFX6-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX6-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s2
+; GFX6-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-SDAG-NEXT: s_endpgm
;
; GFX6-GISEL-LABEL: s_test_fneg_fabs_frexp_exp_f32:
@@ -134,13 +139,14 @@ define amdgpu_kernel void @s_test_fneg_fabs_frexp_exp_f32(ptr addrspace(1) %out,
;
; GFX8-SDAG-LABEL: s_test_fneg_fabs_frexp_exp_f32:
; GFX8-SDAG: ; %bb.0:
-; GFX8-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
+; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s6
-; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX8-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s2
+; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-SDAG-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: s_test_fneg_fabs_frexp_exp_f32:
@@ -164,12 +170,13 @@ define amdgpu_kernel void @s_test_copysign_frexp_exp_f32(ptr addrspace(1) %out,
; GFX6-SDAG-LABEL: s_test_copysign_frexp_exp_f32:
; GFX6-SDAG: ; %bb.0:
; GFX6-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-SDAG-NEXT: s_mov_b32 s6, -1
; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s4
-; GFX6-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX6-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX6-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s2
+; GFX6-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-SDAG-NEXT: s_endpgm
;
; GFX6-GISEL-LABEL: s_test_copysign_frexp_exp_f32:
@@ -188,12 +195,13 @@ define amdgpu_kernel void @s_test_copysign_frexp_exp_f32(ptr addrspace(1) %out,
; GFX8-SDAG-LABEL: s_test_copysign_frexp_exp_f32:
; GFX8-SDAG: ; %bb.0:
; GFX8-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-SDAG-NEXT: s_mov_b32 s6, -1
; GFX8-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX8-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-SDAG-NEXT: s_mov_b32 s2, -1
-; GFX8-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s4
-; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-SDAG-NEXT: s_mov_b32 s4, s0
+; GFX8-SDAG-NEXT: s_mov_b32 s5, s1
+; GFX8-SDAG-NEXT: v_frexp_exp_i32_f32_e32 v0, s2
+; GFX8-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-SDAG-NEXT: s_endpgm
;
; GFX8-GISEL-LABEL: s_test_copysign_frexp_exp_f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
index e2b068e2e9105..88c5a43f25e48 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w32.ll
@@ -21,9 +21,7 @@ declare i32 @llvm.amdgcn.icmp.i1(i1, i1, i32) #0
define amdgpu_kernel void @v_icmp_i32_eq(ptr addrspace(1) %out, i32 %src) {
; SDAG-GFX11-LABEL: v_icmp_i32_eq:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_eq_u32_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -33,9 +31,7 @@ define amdgpu_kernel void @v_icmp_i32_eq(ptr addrspace(1) %out, i32 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i32_eq:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_eq_u32_e64 s2, 0x64, s2
@@ -104,9 +100,7 @@ define amdgpu_kernel void @v_icmp_i32(ptr addrspace(1) %out, i32 %src) {
define amdgpu_kernel void @v_icmp_i32_ne(ptr addrspace(1) %out, i32 %src) {
; SDAG-GFX11-LABEL: v_icmp_i32_ne:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_ne_u32_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -116,9 +110,7 @@ define amdgpu_kernel void @v_icmp_i32_ne(ptr addrspace(1) %out, i32 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i32_ne:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_ne_u32_e64 s2, 0x64, s2
@@ -158,9 +150,7 @@ define amdgpu_kernel void @v_icmp_i32_ne(ptr addrspace(1) %out, i32 %src) {
define amdgpu_kernel void @v_icmp_i32_ugt(ptr addrspace(1) %out, i32 %src) {
; SDAG-GFX11-LABEL: v_icmp_i32_ugt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_lt_u32_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -170,9 +160,7 @@ define amdgpu_kernel void @v_icmp_i32_ugt(ptr addrspace(1) %out, i32 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i32_ugt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_lt_u32_e64 s2, 0x64, s2
@@ -212,9 +200,7 @@ define amdgpu_kernel void @v_icmp_i32_ugt(ptr addrspace(1) %out, i32 %src) {
define amdgpu_kernel void @v_icmp_i32_uge(ptr addrspace(1) %out, i32 %src) {
; SDAG-GFX11-LABEL: v_icmp_i32_uge:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_le_u32_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -224,9 +210,7 @@ define amdgpu_kernel void @v_icmp_i32_uge(ptr addrspace(1) %out, i32 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i32_uge:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_le_u32_e64 s2, 0x64, s2
@@ -266,9 +250,7 @@ define amdgpu_kernel void @v_icmp_i32_uge(ptr addrspace(1) %out, i32 %src) {
define amdgpu_kernel void @v_icmp_i32_ult(ptr addrspace(1) %out, i32 %src) {
; SDAG-GFX11-LABEL: v_icmp_i32_ult:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_gt_u32_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -278,9 +260,7 @@ define amdgpu_kernel void @v_icmp_i32_ult(ptr addrspace(1) %out, i32 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i32_ult:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_gt_u32_e64 s2, 0x64, s2
@@ -320,9 +300,7 @@ define amdgpu_kernel void @v_icmp_i32_ult(ptr addrspace(1) %out, i32 %src) {
define amdgpu_kernel void @v_icmp_i32_ule(ptr addrspace(1) %out, i32 %src) {
; SDAG-GFX11-LABEL: v_icmp_i32_ule:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_ge_u32_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -332,9 +310,7 @@ define amdgpu_kernel void @v_icmp_i32_ule(ptr addrspace(1) %out, i32 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i32_ule:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_ge_u32_e64 s2, 0x64, s2
@@ -374,9 +350,7 @@ define amdgpu_kernel void @v_icmp_i32_ule(ptr addrspace(1) %out, i32 %src) {
define amdgpu_kernel void @v_icmp_i32_sgt(ptr addrspace(1) %out, i32 %src) #1 {
; SDAG-GFX11-LABEL: v_icmp_i32_sgt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_lt_i32_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -386,9 +360,7 @@ define amdgpu_kernel void @v_icmp_i32_sgt(ptr addrspace(1) %out, i32 %src) #1 {
;
; SDAG-GFX10-LABEL: v_icmp_i32_sgt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_lt_i32_e64 s2, 0x64, s2
@@ -428,9 +400,7 @@ define amdgpu_kernel void @v_icmp_i32_sgt(ptr addrspace(1) %out, i32 %src) #1 {
define amdgpu_kernel void @v_icmp_i32_sge(ptr addrspace(1) %out, i32 %src) {
; SDAG-GFX11-LABEL: v_icmp_i32_sge:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_le_i32_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -440,9 +410,7 @@ define amdgpu_kernel void @v_icmp_i32_sge(ptr addrspace(1) %out, i32 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i32_sge:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_le_i32_e64 s2, 0x64, s2
@@ -482,9 +450,7 @@ define amdgpu_kernel void @v_icmp_i32_sge(ptr addrspace(1) %out, i32 %src) {
define amdgpu_kernel void @v_icmp_i32_slt(ptr addrspace(1) %out, i32 %src) {
; SDAG-GFX11-LABEL: v_icmp_i32_slt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_gt_i32_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -494,9 +460,7 @@ define amdgpu_kernel void @v_icmp_i32_slt(ptr addrspace(1) %out, i32 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i32_slt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_gt_i32_e64 s2, 0x64, s2
@@ -536,9 +500,7 @@ define amdgpu_kernel void @v_icmp_i32_slt(ptr addrspace(1) %out, i32 %src) {
define amdgpu_kernel void @v_icmp_i32_sle(ptr addrspace(1) %out, i32 %src) {
; SDAG-GFX11-LABEL: v_icmp_i32_sle:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_ge_i32_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -548,9 +510,7 @@ define amdgpu_kernel void @v_icmp_i32_sle(ptr addrspace(1) %out, i32 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i32_sle:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_ge_i32_e64 s2, 0x64, s2
@@ -1050,9 +1010,7 @@ define amdgpu_kernel void @v_icmp_i64_sle(ptr addrspace(1) %out, i64 %src) {
define amdgpu_kernel void @v_icmp_i16_eq(ptr addrspace(1) %out, i16 %src) {
; SDAG-GFX11-LABEL: v_icmp_i16_eq:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_eq_u16_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1062,9 +1020,7 @@ define amdgpu_kernel void @v_icmp_i16_eq(ptr addrspace(1) %out, i16 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i16_eq:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_eq_u16_e64 s2, 0x64, s2
@@ -1133,9 +1089,7 @@ define amdgpu_kernel void @v_icmp_i16(ptr addrspace(1) %out, i16 %src) {
define amdgpu_kernel void @v_icmp_i16_ne(ptr addrspace(1) %out, i16 %src) {
; SDAG-GFX11-LABEL: v_icmp_i16_ne:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_ne_u16_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1145,9 +1099,7 @@ define amdgpu_kernel void @v_icmp_i16_ne(ptr addrspace(1) %out, i16 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i16_ne:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_ne_u16_e64 s2, 0x64, s2
@@ -1187,9 +1139,7 @@ define amdgpu_kernel void @v_icmp_i16_ne(ptr addrspace(1) %out, i16 %src) {
define amdgpu_kernel void @v_icmp_i16_ugt(ptr addrspace(1) %out, i16 %src) {
; SDAG-GFX11-LABEL: v_icmp_i16_ugt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_lt_u16_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1199,9 +1149,7 @@ define amdgpu_kernel void @v_icmp_i16_ugt(ptr addrspace(1) %out, i16 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i16_ugt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_lt_u16_e64 s2, 0x64, s2
@@ -1241,9 +1189,7 @@ define amdgpu_kernel void @v_icmp_i16_ugt(ptr addrspace(1) %out, i16 %src) {
define amdgpu_kernel void @v_icmp_i16_uge(ptr addrspace(1) %out, i16 %src) {
; SDAG-GFX11-LABEL: v_icmp_i16_uge:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_le_u16_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1253,9 +1199,7 @@ define amdgpu_kernel void @v_icmp_i16_uge(ptr addrspace(1) %out, i16 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i16_uge:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_le_u16_e64 s2, 0x64, s2
@@ -1295,9 +1239,7 @@ define amdgpu_kernel void @v_icmp_i16_uge(ptr addrspace(1) %out, i16 %src) {
define amdgpu_kernel void @v_icmp_i16_ult(ptr addrspace(1) %out, i16 %src) {
; SDAG-GFX11-LABEL: v_icmp_i16_ult:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_gt_u16_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1307,9 +1249,7 @@ define amdgpu_kernel void @v_icmp_i16_ult(ptr addrspace(1) %out, i16 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i16_ult:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_gt_u16_e64 s2, 0x64, s2
@@ -1349,9 +1289,7 @@ define amdgpu_kernel void @v_icmp_i16_ult(ptr addrspace(1) %out, i16 %src) {
define amdgpu_kernel void @v_icmp_i16_ule(ptr addrspace(1) %out, i16 %src) {
; SDAG-GFX11-LABEL: v_icmp_i16_ule:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_ge_u16_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1361,9 +1299,7 @@ define amdgpu_kernel void @v_icmp_i16_ule(ptr addrspace(1) %out, i16 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i16_ule:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_ge_u16_e64 s2, 0x64, s2
@@ -1403,9 +1339,7 @@ define amdgpu_kernel void @v_icmp_i16_ule(ptr addrspace(1) %out, i16 %src) {
define amdgpu_kernel void @v_icmp_i16_sgt(ptr addrspace(1) %out, i16 %src) #1 {
; SDAG-GFX11-LABEL: v_icmp_i16_sgt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_lt_i16_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1415,9 +1349,7 @@ define amdgpu_kernel void @v_icmp_i16_sgt(ptr addrspace(1) %out, i16 %src) #1 {
;
; SDAG-GFX10-LABEL: v_icmp_i16_sgt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_lt_i16_e64 s2, 0x64, s2
@@ -1457,9 +1389,7 @@ define amdgpu_kernel void @v_icmp_i16_sgt(ptr addrspace(1) %out, i16 %src) #1 {
define amdgpu_kernel void @v_icmp_i16_sge(ptr addrspace(1) %out, i16 %src) {
; SDAG-GFX11-LABEL: v_icmp_i16_sge:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_le_i16_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1469,9 +1399,7 @@ define amdgpu_kernel void @v_icmp_i16_sge(ptr addrspace(1) %out, i16 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i16_sge:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_le_i16_e64 s2, 0x64, s2
@@ -1511,9 +1439,7 @@ define amdgpu_kernel void @v_icmp_i16_sge(ptr addrspace(1) %out, i16 %src) {
define amdgpu_kernel void @v_icmp_i16_slt(ptr addrspace(1) %out, i16 %src) {
; SDAG-GFX11-LABEL: v_icmp_i16_slt:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_gt_i16_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1523,9 +1449,7 @@ define amdgpu_kernel void @v_icmp_i16_slt(ptr addrspace(1) %out, i16 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i16_slt:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_gt_i16_e64 s2, 0x64, s2
@@ -1565,9 +1489,7 @@ define amdgpu_kernel void @v_icmp_i16_slt(ptr addrspace(1) %out, i16 %src) {
define amdgpu_kernel void @v_icmp_i16_sle(ptr addrspace(1) %out, i16 %src) {
; SDAG-GFX11-LABEL: v_icmp_i16_sle:
; SDAG-GFX11: ; %bb.0:
-; SDAG-GFX11-NEXT: s_clause 0x1
-; SDAG-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; SDAG-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX11-NEXT: v_cmp_ge_i16_e64 s2, 0x64, s2
; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -1577,9 +1499,7 @@ define amdgpu_kernel void @v_icmp_i16_sle(ptr addrspace(1) %out, i16 %src) {
;
; SDAG-GFX10-LABEL: v_icmp_i16_sle:
; SDAG-GFX10: ; %bb.0:
-; SDAG-GFX10-NEXT: s_clause 0x1
-; SDAG-GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; SDAG-GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; SDAG-GFX10-NEXT: v_mov_b32_e32 v0, 0
; SDAG-GFX10-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX10-NEXT: v_cmp_ge_i16_e64 s2, 0x64, s2
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
index 366b71bae75c9..c108bf50c2f6f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.icmp.w64.ll
@@ -22,46 +22,56 @@ declare i64 @llvm.amdgcn.icmp.i16(i16, i16, i32) #0
declare i64 @llvm.amdgcn.icmp.i1(i1, i1, i32) #0
define amdgpu_kernel void @v_icmp_i32_eq(ptr addrspace(1) %out, i32 %src) {
-; GFX11-LABEL: v_icmp_i32_eq:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_eq_u32_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i32_eq:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_eq_u32_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i32_eq:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_eq_u32_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_eq_u32_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i32_eq:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i32_eq:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i32_eq:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_eq_u32_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i32_eq:
; GISEL-VI: ; %bb.0:
@@ -76,6 +86,19 @@ define amdgpu_kernel void @v_icmp_i32_eq(ptr addrspace(1) %out, i32 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i32_eq:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_eq_u32_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 32)
store i64 %result, ptr addrspace(1) %out
ret void
@@ -124,46 +147,56 @@ define amdgpu_kernel void @v_icmp_i32(ptr addrspace(1) %out, i32 %src) {
}
define amdgpu_kernel void @v_icmp_i32_ne(ptr addrspace(1) %out, i32 %src) {
-; GFX11-LABEL: v_icmp_i32_ne:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u32_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i32_ne:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_ne_u32_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i32_ne:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_ne_u32_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_ne_u32_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i32_ne:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i32_ne:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_ne_u32_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i32_ne:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_ne_u32_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i32_ne:
; GISEL-VI: ; %bb.0:
@@ -178,52 +211,75 @@ define amdgpu_kernel void @v_icmp_i32_ne(ptr addrspace(1) %out, i32 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i32_ne:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_ne_u32_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 33)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i32_ugt(ptr addrspace(1) %out, i32 %src) {
-; GFX11-LABEL: v_icmp_i32_ugt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_lt_u32_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i32_ugt:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_lt_u32_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i32_ugt:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_gt_u32_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_gt_u32_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i32_ugt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_gt_u32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i32_ugt:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_gt_u32_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i32_ugt:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_lt_u32_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i32_ugt:
; GISEL-VI: ; %bb.0:
@@ -238,52 +294,75 @@ define amdgpu_kernel void @v_icmp_i32_ugt(ptr addrspace(1) %out, i32 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i32_ugt:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_gt_u32_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 34)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i32_uge(ptr addrspace(1) %out, i32 %src) {
-; GFX11-LABEL: v_icmp_i32_uge:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_le_u32_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i32_uge:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_le_u32_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i32_uge:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_ge_u32_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_ge_u32_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i32_uge:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_ge_u32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i32_uge:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_ge_u32_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i32_uge:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_le_u32_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i32_uge:
; GISEL-VI: ; %bb.0:
@@ -298,52 +377,75 @@ define amdgpu_kernel void @v_icmp_i32_uge(ptr addrspace(1) %out, i32 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i32_uge:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_ge_u32_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 35)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i32_ult(ptr addrspace(1) %out, i32 %src) {
-; GFX11-LABEL: v_icmp_i32_ult:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_gt_u32_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i32_ult:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_gt_u32_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i32_ult:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_lt_u32_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_lt_u32_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i32_ult:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_lt_u32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i32_ult:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_lt_u32_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i32_ult:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_gt_u32_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i32_ult:
; GISEL-VI: ; %bb.0:
@@ -358,52 +460,75 @@ define amdgpu_kernel void @v_icmp_i32_ult(ptr addrspace(1) %out, i32 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i32_ult:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_lt_u32_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 36)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i32_ule(ptr addrspace(1) %out, i32 %src) {
-; GFX11-LABEL: v_icmp_i32_ule:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ge_u32_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i32_ule:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_ge_u32_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i32_ule:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_le_u32_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_le_u32_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i32_ule:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_le_u32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i32_ule:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_le_u32_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i32_ule:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_ge_u32_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i32_ule:
; GISEL-VI: ; %bb.0:
@@ -418,52 +543,75 @@ define amdgpu_kernel void @v_icmp_i32_ule(ptr addrspace(1) %out, i32 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i32_ule:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_le_u32_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 37)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i32_sgt(ptr addrspace(1) %out, i32 %src) #1 {
-; GFX11-LABEL: v_icmp_i32_sgt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_lt_i32_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i32_sgt:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_lt_i32_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i32_sgt:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_gt_i32_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_gt_i32_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i32_sgt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_gt_i32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i32_sgt:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_gt_i32_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i32_sgt:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_lt_i32_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i32_sgt:
; GISEL-VI: ; %bb.0:
@@ -478,52 +626,75 @@ define amdgpu_kernel void @v_icmp_i32_sgt(ptr addrspace(1) %out, i32 %src) #1 {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i32_sgt:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_gt_i32_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 38)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i32_sge(ptr addrspace(1) %out, i32 %src) {
-; GFX11-LABEL: v_icmp_i32_sge:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_le_i32_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i32_sge:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_le_i32_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i32_sge:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_ge_i32_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_ge_i32_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i32_sge:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_ge_i32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i32_sge:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_ge_i32_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i32_sge:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_le_i32_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i32_sge:
; GISEL-VI: ; %bb.0:
@@ -538,52 +709,75 @@ define amdgpu_kernel void @v_icmp_i32_sge(ptr addrspace(1) %out, i32 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i32_sge:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_ge_i32_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 39)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i32_slt(ptr addrspace(1) %out, i32 %src) {
-; GFX11-LABEL: v_icmp_i32_slt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_gt_i32_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i32_slt:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_gt_i32_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i32_slt:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_lt_i32_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_lt_i32_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i32_slt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_lt_i32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i32_slt:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_lt_i32_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i32_slt:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_gt_i32_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i32_slt:
; GISEL-VI: ; %bb.0:
@@ -598,52 +792,75 @@ define amdgpu_kernel void @v_icmp_i32_slt(ptr addrspace(1) %out, i32 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i32_slt:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_lt_i32_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 40)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i32_sle(ptr addrspace(1) %out, i32 %src) {
-; GFX11-LABEL: v_icmp_i32_sle:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ge_i32_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i32_sle:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_ge_i32_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i32_sle:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_le_i32_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_le_i32_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i32_sle:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_le_i32_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i32_sle:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_le_i32_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i32_sle:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_ge_i32_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i32_sle:
; GISEL-VI: ; %bb.0:
@@ -658,6 +875,19 @@ define amdgpu_kernel void @v_icmp_i32_sle(ptr addrspace(1) %out, i32 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i32_sle:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_le_i32_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i32(i32 %src, i32 100, i32 41)
store i64 %result, ptr addrspace(1) %out
ret void
@@ -1244,46 +1474,56 @@ define amdgpu_kernel void @v_icmp_i64_sle(ptr addrspace(1) %out, i64 %src) {
}
define amdgpu_kernel void @v_icmp_i16_eq(ptr addrspace(1) %out, i16 %src) {
-; GFX11-LABEL: v_icmp_i16_eq:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_eq_u16_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i16_eq:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_eq_u16_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i16_eq:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_eq_u16_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_eq_u16_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i16_eq:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_eq_u16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i16_eq:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_eq_u16_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i16_eq:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_eq_u16_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i16_eq:
; GISEL-VI: ; %bb.0:
@@ -1298,6 +1538,19 @@ define amdgpu_kernel void @v_icmp_i16_eq(ptr addrspace(1) %out, i16 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i16_eq:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_eq_u16_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i16(i16 %src, i16 100, i32 32)
store i64 %result, ptr addrspace(1) %out
ret void
@@ -1346,46 +1599,56 @@ define amdgpu_kernel void @v_icmp_i16(ptr addrspace(1) %out, i16 %src) {
}
define amdgpu_kernel void @v_icmp_i16_ne(ptr addrspace(1) %out, i16 %src) {
-; GFX11-LABEL: v_icmp_i16_ne:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ne_u16_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i16_ne:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_ne_u16_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i16_ne:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_ne_u16_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_ne_u16_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i16_ne:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_ne_u16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i16_ne:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_ne_u16_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i16_ne:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_ne_u16_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i16_ne:
; GISEL-VI: ; %bb.0:
@@ -1400,52 +1663,75 @@ define amdgpu_kernel void @v_icmp_i16_ne(ptr addrspace(1) %out, i16 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i16_ne:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_ne_u16_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i16(i16 %src, i16 100, i32 33)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i16_ugt(ptr addrspace(1) %out, i16 %src) {
-; GFX11-LABEL: v_icmp_i16_ugt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_lt_u16_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i16_ugt:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_lt_u16_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i16_ugt:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_gt_u16_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_gt_u16_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i16_ugt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_gt_u16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i16_ugt:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_gt_u16_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i16_ugt:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_lt_u16_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i16_ugt:
; GISEL-VI: ; %bb.0:
@@ -1460,52 +1746,75 @@ define amdgpu_kernel void @v_icmp_i16_ugt(ptr addrspace(1) %out, i16 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i16_ugt:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_gt_u16_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i16(i16 %src, i16 100, i32 34)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i16_uge(ptr addrspace(1) %out, i16 %src) {
-; GFX11-LABEL: v_icmp_i16_uge:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_le_u16_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i16_uge:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_le_u16_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i16_uge:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_ge_u16_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_ge_u16_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i16_uge:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_ge_u16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i16_uge:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_ge_u16_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i16_uge:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_le_u16_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i16_uge:
; GISEL-VI: ; %bb.0:
@@ -1520,52 +1829,75 @@ define amdgpu_kernel void @v_icmp_i16_uge(ptr addrspace(1) %out, i16 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i16_uge:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_ge_u16_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i16(i16 %src, i16 100, i32 35)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i16_ult(ptr addrspace(1) %out, i16 %src) {
-; GFX11-LABEL: v_icmp_i16_ult:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_gt_u16_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i16_ult:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_gt_u16_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i16_ult:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_lt_u16_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_lt_u16_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i16_ult:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_lt_u16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i16_ult:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_lt_u16_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i16_ult:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_gt_u16_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i16_ult:
; GISEL-VI: ; %bb.0:
@@ -1580,52 +1912,75 @@ define amdgpu_kernel void @v_icmp_i16_ult(ptr addrspace(1) %out, i16 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i16_ult:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_lt_u16_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i16(i16 %src, i16 100, i32 36)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i16_ule(ptr addrspace(1) %out, i16 %src) {
-; GFX11-LABEL: v_icmp_i16_ule:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ge_u16_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i16_ule:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_ge_u16_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i16_ule:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_le_u16_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_le_u16_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i16_ule:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_le_u16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i16_ule:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_le_u16_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i16_ule:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_ge_u16_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i16_ule:
; GISEL-VI: ; %bb.0:
@@ -1640,52 +1995,75 @@ define amdgpu_kernel void @v_icmp_i16_ule(ptr addrspace(1) %out, i16 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i16_ule:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_le_u16_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i16(i16 %src, i16 100, i32 37)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i16_sgt(ptr addrspace(1) %out, i16 %src) #1 {
-; GFX11-LABEL: v_icmp_i16_sgt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_lt_i16_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i16_sgt:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_lt_i16_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i16_sgt:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_gt_i16_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_gt_i16_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i16_sgt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_gt_i16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i16_sgt:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_gt_i16_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i16_sgt:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_lt_i16_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i16_sgt:
; GISEL-VI: ; %bb.0:
@@ -1700,52 +2078,75 @@ define amdgpu_kernel void @v_icmp_i16_sgt(ptr addrspace(1) %out, i16 %src) #1 {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i16_sgt:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_gt_i16_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i16(i16 %src, i16 100, i32 38)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i16_sge(ptr addrspace(1) %out, i16 %src) {
-; GFX11-LABEL: v_icmp_i16_sge:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_le_i16_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i16_sge:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_le_i16_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i16_sge:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_ge_i16_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_ge_i16_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i16_sge:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_ge_i16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i16_sge:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_ge_i16_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i16_sge:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_le_i16_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i16_sge:
; GISEL-VI: ; %bb.0:
@@ -1760,52 +2161,75 @@ define amdgpu_kernel void @v_icmp_i16_sge(ptr addrspace(1) %out, i16 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i16_sge:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_ge_i16_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i16(i16 %src, i16 100, i32 39)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i16_slt(ptr addrspace(1) %out, i16 %src) {
-; GFX11-LABEL: v_icmp_i16_slt:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_gt_i16_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i16_slt:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_gt_i16_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i16_slt:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_lt_i16_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_lt_i16_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i16_slt:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_lt_i16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i16_slt:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_lt_i16_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i16_slt:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_gt_i16_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i16_slt:
; GISEL-VI: ; %bb.0:
@@ -1820,52 +2244,75 @@ define amdgpu_kernel void @v_icmp_i16_slt(ptr addrspace(1) %out, i16 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i16_slt:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_lt_i16_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i16(i16 %src, i16 100, i32 40)
store i64 %result, ptr addrspace(1) %out
ret void
}
define amdgpu_kernel void @v_icmp_i16_sle(ptr addrspace(1) %out, i16 %src) {
-; GFX11-LABEL: v_icmp_i16_sle:
-; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_ge_i16_e64 s[2:3], 0x64, s2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_mov_b32_e32 v0, s2
-; GFX11-NEXT: v_mov_b32_e32 v1, s3
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX11-NEXT: s_endpgm
+; SDAG-GFX11-LABEL: v_icmp_i16_sle:
+; SDAG-GFX11: ; %bb.0:
+; SDAG-GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX11-NEXT: v_cmp_ge_i16_e64 s[2:3], 0x64, s2
+; SDAG-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; SDAG-GFX11-NEXT: s_endpgm
;
; SDAG-VI-LABEL: v_icmp_i16_sle:
; SDAG-VI: ; %bb.0:
-; SDAG-VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; SDAG-VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; SDAG-VI-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, 0x64
; SDAG-VI-NEXT: s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT: v_cmp_le_i16_e64 s[2:3], s2, v0
; SDAG-VI-NEXT: v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT: v_mov_b32_e32 v2, s2
; SDAG-VI-NEXT: v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT: v_mov_b32_e32 v3, s3
+; SDAG-VI-NEXT: v_cmp_le_i16_e64 s[0:1], s2, v2
+; SDAG-VI-NEXT: v_mov_b32_e32 v3, s1
+; SDAG-VI-NEXT: v_mov_b32_e32 v2, s0
; SDAG-VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; SDAG-VI-NEXT: s_endpgm
;
-; GFX9-LABEL: v_icmp_i16_sle:
-; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x64
-; GFX9-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_cmp_le_i16_e64 s[2:3], s2, v0
-; GFX9-NEXT: v_mov_b32_e32 v0, s2
-; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
-; GFX9-NEXT: s_endpgm
+; SDAG-GFX9-LABEL: v_icmp_i16_sle:
+; SDAG-GFX9: ; %bb.0:
+; SDAG-GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; SDAG-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; SDAG-GFX9-NEXT: v_cmp_le_i16_e64 s[2:3], s2, v0
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; SDAG-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; SDAG-GFX9-NEXT: s_endpgm
+;
+; GISEL-GFX11-LABEL: v_icmp_i16_sle:
+; GISEL-GFX11: ; %bb.0:
+; GISEL-GFX11-NEXT: s_clause 0x1
+; GISEL-GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GISEL-GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX11-NEXT: v_cmp_ge_i16_e64 s[2:3], 0x64, s2
+; GISEL-GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GISEL-GFX11-NEXT: s_endpgm
;
; GISEL-VI-LABEL: v_icmp_i16_sle:
; GISEL-VI: ; %bb.0:
@@ -1880,6 +2327,19 @@ define amdgpu_kernel void @v_icmp_i16_sle(ptr addrspace(1) %out, i16 %src) {
; GISEL-VI-NEXT: v_mov_b32_e32 v2, s0
; GISEL-VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GISEL-VI-NEXT: s_endpgm
+;
+; GISEL-GFX9-LABEL: v_icmp_i16_sle:
+; GISEL-GFX9: ; %bb.0:
+; GISEL-GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GISEL-GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, 0x64
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v2, 0
+; GISEL-GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GISEL-GFX9-NEXT: v_cmp_le_i16_e64 s[2:3], s2, v0
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GISEL-GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GISEL-GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
+; GISEL-GFX9-NEXT: s_endpgm
%result = call i64 @llvm.amdgcn.icmp.i16(i16 %src, i16 100, i32 41)
store i64 %result, ptr addrspace(1) %out
ret void
@@ -1907,14 +2367,14 @@ define amdgpu_kernel void @v_icmp_i1_ne0(ptr addrspace(1) %out, i32 %a, i32 %b)
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_cmp_gt_u32 s2, 1
-; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
-; VI-NEXT: s_cmp_gt_u32 s3, 2
-; VI-NEXT: s_cselect_b64 s[2:3], -1, 0
-; VI-NEXT: s_and_b64 s[2:3], s[4:5], s[2:3]
; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v3, s3
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: s_cmp_gt_u32 s3, 2
+; VI-NEXT: s_cselect_b64 s[2:3], -1, 0
+; VI-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
+; VI-NEXT: v_mov_b32_e32 v3, s1
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
index 700fc6db8549f..e87a9096b9cfa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.gfx942.ll
@@ -1248,12 +1248,13 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x32_f16(ptr addrspace(1) %arg, <
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s11
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[12:13]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[14:15]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s6
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s6
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX942-SDAG-NEXT: s_nop 1
; GFX942-SDAG-NEXT: v_smfmac_f32_16x16x32_f16 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
@@ -1288,12 +1289,13 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x32_f16(ptr addrspace(1) %arg, <
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX950-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s11
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[12:13]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[14:15]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s6
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s6
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_smfmac_f32_16x16x32_f16 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
@@ -1377,8 +1379,9 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x16_f16(ptr addrspace(1) %arg, <
; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
; GFX942-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s18
; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s19
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[20:21]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[22:23]
; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s24
@@ -1435,8 +1438,9 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x16_f16(ptr addrspace(1) %arg, <
; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
; GFX950-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s18
; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s19
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[20:21]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[22:23]
; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s24
@@ -1650,12 +1654,13 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x32_bf16(ptr addrspace(1) %arg,
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s11
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[12:13]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[14:15]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s6
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s6
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX942-SDAG-NEXT: s_nop 1
; GFX942-SDAG-NEXT: v_smfmac_f32_16x16x32_bf16 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
@@ -1690,12 +1695,13 @@ define amdgpu_kernel void @test_smfmac_f32_16x16x32_bf16(ptr addrspace(1) %arg,
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX950-SDAG-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s11
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[12:13]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[14:15]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s6
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s6
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_smfmac_f32_16x16x32_bf16 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
@@ -1779,8 +1785,9 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x16_bf16(ptr addrspace(1) %arg,
; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
; GFX942-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s18
; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s19
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[20:21]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[22:23]
; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s24
@@ -1837,8 +1844,9 @@ define amdgpu_kernel void @test_smfmac_f32_32x32x16_bf16(ptr addrspace(1) %arg,
; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
; GFX950-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b64_e32 v[22:23], s[18:19]
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s18
; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s19
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[18:19], s[20:21]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[20:21], s[22:23]
; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s24
@@ -2047,96 +2055,88 @@ bb:
define amdgpu_kernel void @test_smfmac_i32_16x16x64_i8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 {
; GFX942-SDAG-LABEL: test_smfmac_i32_16x16x64_i8:
; GFX942-SDAG: ; %bb.0: ; %bb
-; GFX942-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c
-; GFX942-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-SDAG-NEXT: s_load_dword s4, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s8
-; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s9
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s14
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s14
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX942-SDAG-NEXT: s_nop 1
; GFX942-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
; GFX942-SDAG-NEXT: s_nop 6
-; GFX942-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[8:9]
; GFX942-SDAG-NEXT: s_endpgm
;
; GFX942-GISEL-LABEL: test_smfmac_i32_16x16x64_i8:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX942-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0
-; GFX942-GISEL-NEXT: s_mov_b32 s4, s2
-; GFX942-GISEL-NEXT: s_mov_b32 s5, s3
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX942-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, s6
; GFX942-GISEL-NEXT: s_nop 1
; GFX942-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX942-GISEL-NEXT: s_nop 5
-; GFX942-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9]
; GFX942-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: test_smfmac_i32_16x16x64_i8:
; GFX950-SDAG: ; %bb.0: ; %bb
-; GFX950-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c
-; GFX950-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-SDAG-NEXT: s_load_dword s4, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
-; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s14
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s14
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_smfmac_i32_16x16x64_i8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
; GFX950-SDAG-NEXT: s_nop 7
-; GFX950-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7]
+; GFX950-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[8:9]
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: test_smfmac_i32_16x16x64_i8:
; GFX950-GISEL: ; %bb.0: ; %bb
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX950-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44
+; GFX950-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s3
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX950-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s6
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_smfmac_i32_16x16x64_i8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2
; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX950-GISEL-NEXT: s_nop 6
-; GFX950-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9]
; GFX950-GISEL-NEXT: s_endpgm
; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_i8:
; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb
@@ -2258,19 +2258,19 @@ bb:
define amdgpu_kernel void @test_smfmac_i32_32x32x32_i8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 {
; GFX942-SDAG-LABEL: test_smfmac_i32_32x32x32_i8:
; GFX942-SDAG: ; %bb.0: ; %bb
-; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
-; GFX942-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s20
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s21
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s22
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s22
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, s23
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s24
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -2282,28 +2282,24 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_i8(ptr addrspace(1) %arg, <2
; GFX942-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX942-SDAG-NEXT: s_nop 9
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-SDAG-NEXT: s_endpgm
;
; GFX942-GISEL-LABEL: test_smfmac_i32_32x32x32_i8:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c
-; GFX942-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-GISEL-NEXT: s_mov_b32 s20, s18
-; GFX942-GISEL-NEXT: s_mov_b32 s21, s19
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v22, s26
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v22, s24
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -2315,27 +2311,27 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_i8(ptr addrspace(1) %arg, <2
; GFX942-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX942-GISEL-NEXT: s_nop 9
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX942-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: test_smfmac_i32_32x32x32_i8:
; GFX950-SDAG: ; %bb.0: ; %bb
-; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
-; GFX950-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s21
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s22
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s22
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s23
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s24
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -2347,28 +2343,24 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_i8(ptr addrspace(1) %arg, <2
; GFX950-SDAG-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2
; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX950-SDAG-NEXT: s_nop 10
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: test_smfmac_i32_32x32x32_i8:
; GFX950-GISEL: ; %bb.0: ; %bb
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c
-; GFX950-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44
+; GFX950-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX950-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
-; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-GISEL-NEXT: s_mov_b32 s20, s18
-; GFX950-GISEL-NEXT: s_mov_b32 s21, s19
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, s26
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, s24
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -2380,10 +2372,10 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_i8(ptr addrspace(1) %arg, <2
; GFX950-GISEL-NEXT: v_smfmac_i32_32x32x32_i8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2
; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX950-GISEL-NEXT: s_nop 10
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX950-GISEL-NEXT: s_endpgm
; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_i8:
; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb
@@ -2561,96 +2553,88 @@ bb:
define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 {
; GFX942-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8:
; GFX942-SDAG: ; %bb.0: ; %bb
-; GFX942-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c
-; GFX942-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-SDAG-NEXT: s_load_dword s4, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s8
-; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s9
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s14
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s14
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX942-SDAG-NEXT: s_nop 1
; GFX942-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
; GFX942-SDAG-NEXT: s_nop 6
-; GFX942-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[8:9]
; GFX942-SDAG-NEXT: s_endpgm
;
; GFX942-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX942-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0
-; GFX942-GISEL-NEXT: s_mov_b32 s4, s2
-; GFX942-GISEL-NEXT: s_mov_b32 s5, s3
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX942-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, s6
; GFX942-GISEL-NEXT: s_nop 1
; GFX942-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX942-GISEL-NEXT: s_nop 5
-; GFX942-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9]
; GFX942-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8:
; GFX950-SDAG: ; %bb.0: ; %bb
-; GFX950-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c
-; GFX950-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-SDAG-NEXT: s_load_dword s4, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
-; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s14
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s14
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
; GFX950-SDAG-NEXT: s_nop 7
-; GFX950-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7]
+; GFX950-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[8:9]
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_bf8:
; GFX950-GISEL: ; %bb.0: ; %bb
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX950-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44
+; GFX950-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s3
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX950-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s6
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2
; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX950-GISEL-NEXT: s_nop 6
-; GFX950-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9]
; GFX950-GISEL-NEXT: s_endpgm
; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_bf8:
; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb
@@ -2772,96 +2756,88 @@ bb:
define amdgpu_kernel void @test_smfmac_i32_16x16x64_bf8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 {
; GFX942-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8:
; GFX942-SDAG: ; %bb.0: ; %bb
-; GFX942-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c
-; GFX942-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-SDAG-NEXT: s_load_dword s4, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s8
-; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s9
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s14
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s14
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX942-SDAG-NEXT: s_nop 1
; GFX942-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
; GFX942-SDAG-NEXT: s_nop 6
-; GFX942-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[8:9]
; GFX942-SDAG-NEXT: s_endpgm
;
; GFX942-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX942-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0
-; GFX942-GISEL-NEXT: s_mov_b32 s4, s2
-; GFX942-GISEL-NEXT: s_mov_b32 s5, s3
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX942-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, s6
; GFX942-GISEL-NEXT: s_nop 1
; GFX942-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX942-GISEL-NEXT: s_nop 5
-; GFX942-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9]
; GFX942-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8:
; GFX950-SDAG: ; %bb.0: ; %bb
-; GFX950-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c
-; GFX950-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-SDAG-NEXT: s_load_dword s4, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
-; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s14
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s14
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
; GFX950-SDAG-NEXT: s_nop 7
-; GFX950-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7]
+; GFX950-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[8:9]
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: test_smfmac_i32_16x16x64_bf8_fp8:
; GFX950-GISEL: ; %bb.0: ; %bb
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX950-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44
+; GFX950-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s3
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX950-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s6
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_smfmac_f32_16x16x64_bf8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2
; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX950-GISEL-NEXT: s_nop 6
-; GFX950-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9]
; GFX950-GISEL-NEXT: s_endpgm
; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_bf8_fp8:
; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb
@@ -2983,96 +2959,88 @@ bb:
define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 {
; GFX942-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8:
; GFX942-SDAG: ; %bb.0: ; %bb
-; GFX942-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c
-; GFX942-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-SDAG-NEXT: s_load_dword s4, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s8
-; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s9
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s14
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s14
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX942-SDAG-NEXT: s_nop 1
; GFX942-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
; GFX942-SDAG-NEXT: s_nop 6
-; GFX942-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[8:9]
; GFX942-SDAG-NEXT: s_endpgm
;
; GFX942-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX942-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0
-; GFX942-GISEL-NEXT: s_mov_b32 s4, s2
-; GFX942-GISEL-NEXT: s_mov_b32 s5, s3
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX942-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, s6
; GFX942-GISEL-NEXT: s_nop 1
; GFX942-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX942-GISEL-NEXT: s_nop 5
-; GFX942-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9]
; GFX942-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8:
; GFX950-SDAG: ; %bb.0: ; %bb
-; GFX950-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c
-; GFX950-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-SDAG-NEXT: s_load_dword s4, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
-; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s14
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s14
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
; GFX950-SDAG-NEXT: s_nop 7
-; GFX950-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7]
+; GFX950-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[8:9]
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_bf8:
; GFX950-GISEL: ; %bb.0: ; %bb
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX950-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44
+; GFX950-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s3
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX950-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s6
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_bf8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2
; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX950-GISEL-NEXT: s_nop 6
-; GFX950-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9]
; GFX950-GISEL-NEXT: s_endpgm
; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_bf8:
; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb
@@ -3194,96 +3162,88 @@ bb:
define amdgpu_kernel void @test_smfmac_i32_16x16x64_fp8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 {
; GFX942-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8:
; GFX942-SDAG: ; %bb.0: ; %bb
-; GFX942-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c
-; GFX942-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX942-SDAG-NEXT: s_load_dword s4, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s8
-; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s9
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX942-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, s14
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s14
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX942-SDAG-NEXT: s_nop 1
; GFX942-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
; GFX942-SDAG-NEXT: s_nop 6
-; GFX942-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[8:9]
; GFX942-SDAG-NEXT: s_endpgm
;
; GFX942-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX942-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0
-; GFX942-GISEL-NEXT: s_mov_b32 s4, s2
-; GFX942-GISEL-NEXT: s_mov_b32 s5, s3
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX942-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX942-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, s6
; GFX942-GISEL-NEXT: s_nop 1
; GFX942-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX942-GISEL-NEXT: s_nop 5
-; GFX942-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9]
; GFX942-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8:
; GFX950-SDAG: ; %bb.0: ; %bb
-; GFX950-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x2c
-; GFX950-SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX950-SDAG-NEXT: v_mov_b32_e32 v0, 0
+; GFX950-SDAG-NEXT: s_load_dword s4, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s8
-; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[6:7], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s9
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s10
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s11
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v10, s10
+; GFX950-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v11, s11
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v2, s12
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v3, s13
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v4, s14
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[8:9], s[2:3]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s13
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s14
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v5, s15
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v1, s4
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[0:1]
; GFX950-SDAG-NEXT: s_nop 1
; GFX950-SDAG-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[6:9], v[10:11], v[2:5], v1 cbsz:1 abid:2
; GFX950-SDAG-NEXT: s_nop 7
-; GFX950-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[6:7]
+; GFX950-SDAG-NEXT: global_store_dwordx4 v0, v[6:9], s[8:9]
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: test_smfmac_i32_16x16x64_fp8_fp8:
; GFX950-GISEL: ; %bb.0: ; %bb
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[12:13], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x3c
-; GFX950-GISEL-NEXT: s_load_dword s14, s[4:5], 0x44
+; GFX950-GISEL-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x24
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[0:1]
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[8:11], s[12:13], 0x0
-; GFX950-GISEL-NEXT: s_mov_b32 s4, s2
-; GFX950-GISEL-NEXT: s_mov_b32 s5, s3
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[4:5]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[6:7]
+; GFX950-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX950-GISEL-NEXT: s_load_dword s6, s[4:5], 0x44
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[10:11]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[12:13]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[14:15]
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[8:9]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[10:11]
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s14
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[10:11], s[2:3]
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[8:9], s[0:1]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v6, s6
; GFX950-GISEL-NEXT: s_nop 1
; GFX950-GISEL-NEXT: v_smfmac_f32_16x16x64_fp8_fp8 v[8:11], v[4:5], v[0:3], v6 cbsz:1 abid:2
; GFX950-GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX950-GISEL-NEXT: s_nop 6
-; GFX950-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[12:13]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v0, v[8:11], s[8:9]
; GFX950-GISEL-NEXT: s_endpgm
; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_16x16x64_fp8_fp8:
; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb
@@ -3405,19 +3365,19 @@ bb:
define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 {
; GFX942-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8:
; GFX942-SDAG: ; %bb.0: ; %bb
-; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
-; GFX942-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s20
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s21
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s22
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s22
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, s23
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s24
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -3429,28 +3389,24 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_bf8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX942-SDAG-NEXT: s_nop 9
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-SDAG-NEXT: s_endpgm
;
; GFX942-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c
-; GFX942-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-GISEL-NEXT: s_mov_b32 s20, s18
-; GFX942-GISEL-NEXT: s_mov_b32 s21, s19
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v22, s26
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v22, s24
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -3462,27 +3418,27 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_bf8(ptr addrspace(1) %ar
; GFX942-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX942-GISEL-NEXT: s_nop 9
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX942-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8:
; GFX950-SDAG: ; %bb.0: ; %bb
-; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
-; GFX950-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s21
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s22
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s22
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s23
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s24
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -3494,28 +3450,24 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_bf8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2
; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX950-SDAG-NEXT: s_nop 10
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_bf8:
; GFX950-GISEL: ; %bb.0: ; %bb
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c
-; GFX950-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44
+; GFX950-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX950-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
-; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-GISEL-NEXT: s_mov_b32 s20, s18
-; GFX950-GISEL-NEXT: s_mov_b32 s21, s19
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, s26
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, s24
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -3527,10 +3479,10 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_bf8(ptr addrspace(1) %ar
; GFX950-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2
; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX950-GISEL-NEXT: s_nop 10
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX950-GISEL-NEXT: s_endpgm
; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_bf8:
; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb
@@ -3708,19 +3660,19 @@ bb:
define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 {
; GFX942-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8:
; GFX942-SDAG: ; %bb.0: ; %bb
-; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
-; GFX942-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s20
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s21
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s22
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s22
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, s23
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s24
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -3732,28 +3684,24 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_fp8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX942-SDAG-NEXT: s_nop 9
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-SDAG-NEXT: s_endpgm
;
; GFX942-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c
-; GFX942-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-GISEL-NEXT: s_mov_b32 s20, s18
-; GFX942-GISEL-NEXT: s_mov_b32 s21, s19
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v22, s26
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v22, s24
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -3765,27 +3713,27 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_fp8(ptr addrspace(1) %ar
; GFX942-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX942-GISEL-NEXT: s_nop 9
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX942-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8:
; GFX950-SDAG: ; %bb.0: ; %bb
-; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
-; GFX950-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s21
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s22
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s22
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s23
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s24
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -3797,28 +3745,24 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_fp8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2
; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX950-SDAG-NEXT: s_nop 10
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: test_smfmac_i32_32x32x32_bf8_fp8:
; GFX950-GISEL: ; %bb.0: ; %bb
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c
-; GFX950-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44
+; GFX950-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX950-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
-; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-GISEL-NEXT: s_mov_b32 s20, s18
-; GFX950-GISEL-NEXT: s_mov_b32 s21, s19
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, s26
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, s24
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -3830,10 +3774,10 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_bf8_fp8(ptr addrspace(1) %ar
; GFX950-GISEL-NEXT: v_smfmac_f32_32x32x32_bf8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2
; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX950-GISEL-NEXT: s_nop 10
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX950-GISEL-NEXT: s_endpgm
; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_bf8_fp8:
; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb
@@ -4011,19 +3955,19 @@ bb:
define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_bf8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 {
; GFX942-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8:
; GFX942-SDAG: ; %bb.0: ; %bb
-; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
-; GFX942-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s20
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s21
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s22
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s22
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, s23
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s24
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -4035,28 +3979,24 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_bf8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX942-SDAG-NEXT: s_nop 9
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-SDAG-NEXT: s_endpgm
;
; GFX942-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c
-; GFX942-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-GISEL-NEXT: s_mov_b32 s20, s18
-; GFX942-GISEL-NEXT: s_mov_b32 s21, s19
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v22, s26
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v22, s24
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -4068,27 +4008,27 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_bf8(ptr addrspace(1) %ar
; GFX942-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX942-GISEL-NEXT: s_nop 9
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX942-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8:
; GFX950-SDAG: ; %bb.0: ; %bb
-; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
-; GFX950-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s21
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s22
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s22
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s23
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s24
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -4100,28 +4040,24 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_bf8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2
; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX950-SDAG-NEXT: s_nop 10
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_bf8:
; GFX950-GISEL: ; %bb.0: ; %bb
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c
-; GFX950-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44
+; GFX950-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX950-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
-; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-GISEL-NEXT: s_mov_b32 s20, s18
-; GFX950-GISEL-NEXT: s_mov_b32 s21, s19
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, s26
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, s24
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -4133,10 +4069,10 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_bf8(ptr addrspace(1) %ar
; GFX950-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_bf8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2
; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX950-GISEL-NEXT: s_nop 10
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX950-GISEL-NEXT: s_endpgm
; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_bf8:
; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb
@@ -4314,19 +4250,19 @@ bb:
define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_fp8(ptr addrspace(1) %arg, <2 x i32> %a, <4 x i32> %b, i32 %idx) #0 {
; GFX942-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8:
; GFX942-SDAG: ; %bb.0: ; %bb
-; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
-; GFX942-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX942-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; GFX942-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v18, s20
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v19, s21
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v20, s22
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s22
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v21, s23
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, s24
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -4338,28 +4274,24 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_fp8(ptr addrspace(1) %ar
; GFX942-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX942-SDAG-NEXT: s_nop 9
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX942-SDAG-NEXT: s_endpgm
;
; GFX942-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8:
; GFX942-GISEL: ; %bb.0: ; %bb
-; GFX942-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
-; GFX942-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c
-; GFX942-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44
+; GFX942-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX942-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
-; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX942-GISEL-NEXT: s_mov_b32 s20, s18
-; GFX942-GISEL-NEXT: s_mov_b32 s21, s19
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX942-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v22, s26
+; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v22, s24
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX942-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX942-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -4371,27 +4303,27 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_fp8(ptr addrspace(1) %ar
; GFX942-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX942-GISEL-NEXT: s_nop 9
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX942-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX942-GISEL-NEXT: s_endpgm
;
; GFX950-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8:
; GFX950-SDAG: ; %bb.0: ; %bb
-; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x2c
-; GFX950-SDAG-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX950-SDAG-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s16
-; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s17
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s18
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s19
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v22, s18
+; GFX950-SDAG-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v23, s19
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v18, s20
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v19, s21
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v20, s22
; GFX950-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s21
-; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s22
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v21, s23
+; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, s24
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX950-SDAG-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -4403,28 +4335,24 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_fp8(ptr addrspace(1) %ar
; GFX950-SDAG-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[22:23], v[18:21], v16 cbsz:1 abid:2
; GFX950-SDAG-NEXT: v_mov_b32_e32 v16, 0
; GFX950-SDAG-NEXT: s_nop 10
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX950-SDAG-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
; GFX950-SDAG-NEXT: s_endpgm
;
; GFX950-GISEL-LABEL: test_smfmac_i32_32x32x32_fp8_fp8:
; GFX950-GISEL: ; %bb.0: ; %bb
-; GFX950-GISEL-NEXT: s_load_dwordx4 s[16:19], s[4:5], 0x2c
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[24:25], s[4:5], 0x24
-; GFX950-GISEL-NEXT: s_load_dwordx2 s[22:23], s[4:5], 0x3c
-; GFX950-GISEL-NEXT: s_load_dword s26, s[4:5], 0x44
+; GFX950-GISEL-NEXT: s_load_dwordx8 s[16:23], s[4:5], 0x24
+; GFX950-GISEL-NEXT: s_load_dword s24, s[4:5], 0x44
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[16:17]
-; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[24:25], 0x0
-; GFX950-GISEL-NEXT: s_mov_b32 s20, s18
-; GFX950-GISEL-NEXT: s_mov_b32 s21, s19
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[20:21], s[18:19]
+; GFX950-GISEL-NEXT: s_load_dwordx16 s[0:15], s[16:17], 0x0
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[16:17], s[20:21]
-; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, s26
+; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
+; GFX950-GISEL-NEXT: v_mov_b32_e32 v22, s24
; GFX950-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
-; GFX950-GISEL-NEXT: v_mov_b64_e32 v[18:19], s[22:23]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
; GFX950-GISEL-NEXT: v_mov_b64_e32 v[6:7], s[6:7]
@@ -4436,10 +4364,10 @@ define amdgpu_kernel void @test_smfmac_i32_32x32x32_fp8_fp8(ptr addrspace(1) %ar
; GFX950-GISEL-NEXT: v_smfmac_f32_32x32x32_fp8_fp8 v[0:15], v[20:21], v[16:19], v22 cbsz:1 abid:2
; GFX950-GISEL-NEXT: v_mov_b32_e32 v16, 0
; GFX950-GISEL-NEXT: s_nop 10
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[24:25]
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[24:25] offset:16
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[24:25] offset:32
-; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[24:25] offset:48
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[0:3], s[16:17]
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[4:7], s[16:17] offset:16
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[8:11], s[16:17] offset:32
+; GFX950-GISEL-NEXT: global_store_dwordx4 v16, v[12:15], s[16:17] offset:48
; GFX950-GISEL-NEXT: s_endpgm
; GFX942-AGPRCD-SDAG-LABEL: test_smfmac_i32_32x32x32_fp8_fp8:
; GFX942-AGPRCD-SDAG: ; %bb.0: ; %bb
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
index 20f6f5021d5ef..9cfa8003de7a4 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mfma.scale.f32.16x16x128.f8f6f4.ll
@@ -1222,8 +1222,8 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__inlineimm:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v21, -2
; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
; SDAG-NEXT: v_mov_b32_e32 v20, 0
@@ -1236,7 +1236,6 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v5, s13
; SDAG-NEXT: v_mov_b32_e32 v6, s14
; SDAG-NEXT: v_mov_b32_e32 v7, s15
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; SDAG-NEXT: v_mov_b32_e32 v8, s16
; SDAG-NEXT: v_mov_b32_e32 v9, s17
; SDAG-NEXT: v_mov_b32_e32 v10, s18
@@ -1245,7 +1244,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v13, s21
; SDAG-NEXT: v_mov_b32_e32 v14, s22
; SDAG-NEXT: v_mov_b32_e32 v15, s23
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; SDAG-NEXT: v_mov_b32_e32 v16, s0
+; SDAG-NEXT: v_mov_b32_e32 v17, s1
+; SDAG-NEXT: v_mov_b32_e32 v18, s2
+; SDAG-NEXT: v_mov_b32_e32 v19, s3
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
@@ -1285,8 +1287,8 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_kimm__scaleB__FP_literal:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v21, 1.0
; SDAG-NEXT: v_mov_b32_e32 v22, 0x41
; SDAG-NEXT: v_mov_b32_e32 v20, 0
@@ -1299,7 +1301,6 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v5, s13
; SDAG-NEXT: v_mov_b32_e32 v6, s14
; SDAG-NEXT: v_mov_b32_e32 v7, s15
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; SDAG-NEXT: v_mov_b32_e32 v8, s16
; SDAG-NEXT: v_mov_b32_e32 v9, s17
; SDAG-NEXT: v_mov_b32_e32 v10, s18
@@ -1308,7 +1309,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v13, s21
; SDAG-NEXT: v_mov_b32_e32 v14, s22
; SDAG-NEXT: v_mov_b32_e32 v15, s23
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; SDAG-NEXT: v_mov_b32_e32 v16, s0
+; SDAG-NEXT: v_mov_b32_e32 v17, s1
+; SDAG-NEXT: v_mov_b32_e32 v18, s2
+; SDAG-NEXT: v_mov_b32_e32 v19, s3
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
@@ -1348,8 +1352,8 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__inline_imm:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v21, -2
; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
@@ -1362,7 +1366,6 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v5, s13
; SDAG-NEXT: v_mov_b32_e32 v6, s14
; SDAG-NEXT: v_mov_b32_e32 v7, s15
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; SDAG-NEXT: v_mov_b32_e32 v8, s16
; SDAG-NEXT: v_mov_b32_e32 v9, s17
; SDAG-NEXT: v_mov_b32_e32 v10, s18
@@ -1371,7 +1374,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v13, s21
; SDAG-NEXT: v_mov_b32_e32 v14, s22
; SDAG-NEXT: v_mov_b32_e32 v15, s23
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; SDAG-NEXT: v_mov_b32_e32 v16, s0
+; SDAG-NEXT: v_mov_b32_e32 v17, s1
+; SDAG-NEXT: v_mov_b32_e32 v18, s2
+; SDAG-NEXT: v_mov_b32_e32 v19, s3
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
@@ -1411,8 +1417,8 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-LABEL: test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA_FP_literal__scaleB__FP_literal:
; SDAG: ; %bb.0:
; SDAG-NEXT: s_load_dwordx16 s[8:23], s[4:5], 0x0
-; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x50
+; SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x40
; SDAG-NEXT: v_mov_b32_e32 v21, 0.15915494
; SDAG-NEXT: v_mov_b32_e32 v22, 1.0
; SDAG-NEXT: v_mov_b32_e32 v20, 0
@@ -1425,7 +1431,6 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v5, s13
; SDAG-NEXT: v_mov_b32_e32 v6, s14
; SDAG-NEXT: v_mov_b32_e32 v7, s15
-; SDAG-NEXT: v_mov_b64_e32 v[18:19], s[2:3]
; SDAG-NEXT: v_mov_b32_e32 v8, s16
; SDAG-NEXT: v_mov_b32_e32 v9, s17
; SDAG-NEXT: v_mov_b32_e32 v10, s18
@@ -1434,7 +1439,10 @@ define amdgpu_kernel void @test_mfma_scale_f32_16x16x128_f8f6f4__vgprcd___scaleA
; SDAG-NEXT: v_mov_b32_e32 v13, s21
; SDAG-NEXT: v_mov_b32_e32 v14, s22
; SDAG-NEXT: v_mov_b32_e32 v15, s23
-; SDAG-NEXT: v_mov_b64_e32 v[16:17], s[0:1]
+; SDAG-NEXT: v_mov_b32_e32 v16, s0
+; SDAG-NEXT: v_mov_b32_e32 v17, s1
+; SDAG-NEXT: v_mov_b32_e32 v18, s2
+; SDAG-NEXT: v_mov_b32_e32 v19, s3
; SDAG-NEXT: s_nop 1
; SDAG-NEXT: v_mfma_scale_f32_16x16x128_f8f6f4 v[0:3], v[0:7], v[8:15], v[16:19], v22, v21 op_sel:[1,1,0] op_sel_hi:[1,0,0]
; SDAG-NEXT: s_nop 11
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
index 97cd7e5874a03..07e99ed51407c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.mov.dpp.ll
@@ -9,7 +9,9 @@
; VI-LABEL: {{^}}dpp_test:
; VI: v_mov_b32_e32 v0, s{{[0-9]+}}
; VI-NOOPT: v_mov_b32_e32 v1, s{{[0-9]+}}
-; PREGFX10: s_nop 1
+; PREGFX10-OPT-DAG: s_mov_b32 s{{[0-9]+}}, s0
+; PREGFX10-OPT-DAG: s_mov_b32 s{{[0-9]+}}, s1
+; PREGFX10-NOOPT: s_nop 1
; VI-OPT: v_mov_b32_dpp v0, v0 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:1 ; encoding: [0xfa,0x02,0x00,0x7e,0x00,0x01,0x08,0x11]
; VI-NOOPT: v_mov_b32_dpp v0, v1 quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:1 ; encoding: [0xfa,0x02,0x00,0x7e,0x01,0x01,0x08,0x11]
define amdgpu_kernel void @dpp_test(ptr addrspace(1) %out, i32 %in) {
@@ -21,7 +23,9 @@ define amdgpu_kernel void @dpp_test(ptr addrspace(1) %out, i32 %in) {
; VI-LABEL: {{^}}dpp_wait_states:
; VI-NOOPT: v_mov_b32_e32 [[VGPR1:v[0-9]+]], s{{[0-9]+}}
; VI: v_mov_b32_e32 [[VGPR0:v[0-9]+]], s{{[0-9]+}}
-; PREGFX10: s_nop 1
+; PREGFX10-OPT-DAG: s_mov_b32 s{{[0-9]+}}, s0
+; PREGFX10-OPT-DAG: s_mov_b32 s{{[0-9]+}}, s1
+; PREGFX10-NOOPT: s_nop 1
; VI-OPT: v_mov_b32_dpp [[VGPR0]], [[VGPR0]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:1
; VI-NOOPT: v_mov_b32_dpp [[VGPR1]], [[VGPR0]] quad_perm:[1,0,0,0] row_mask:0x1 bank_mask:0x1 bound_ctrl:
; PREGFX10: s_nop 1
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ll
index 7489d56e68e6c..e389cc6846692 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ll
@@ -7,9 +7,7 @@ declare i32 @llvm.amdgcn.workitem.id.x()
define amdgpu_kernel void @test_s_i32(ptr addrspace(1) %out, i32 %src0) {
; GFX11-SDAG-LABEL: test_s_i32:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll
index 672b658659824..132571f1bd3b2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.permlane64.ptr.ll
@@ -39,9 +39,7 @@ define amdgpu_kernel void @test_v3p0(ptr addrspace(1) %out, <3 x ptr> %src0) {
define amdgpu_kernel void @test_p3(ptr addrspace(1) %out, ptr addrspace(3) %src0) {
; GFX11-SDAG-LABEL: test_p3:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -70,9 +68,7 @@ define amdgpu_kernel void @test_v3p3(ptr addrspace(1) %out, <3 x ptr addrspace(3
define amdgpu_kernel void @test_p5(ptr addrspace(1) %out, ptr addrspace(5) %src0) {
; GFX11-SDAG-LABEL: test_p5:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -101,9 +97,7 @@ define amdgpu_kernel void @test_v3p5(ptr addrspace(1) %out, <3 x ptr addrspace(5
define amdgpu_kernel void @test_p6(ptr addrspace(1) %out, ptr addrspace(6) %src0) {
; GFX11-SDAG-LABEL: test_p6:
; GFX11-SDAG: ; %bb.0:
-; GFX11-SDAG-NEXT: s_clause 0x1
-; GFX11-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX11-SDAG-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.format.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.format.d16.ll
index 79fba612d72f2..c12d098644c06 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.format.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.buffer.store.format.d16.ll
@@ -50,9 +50,7 @@ main_body:
define amdgpu_kernel void @buffer_store_format_d16_xyz(<4 x i32> %rsrc, <4 x half> %data, i32 %voffset) {
; UNPACKED-LABEL: buffer_store_format_d16_xyz:
; UNPACKED: ; %bb.0: ; %main_body
-; UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
; UNPACKED-NEXT: s_lshr_b32 s7, s4, 16
@@ -66,9 +64,7 @@ define amdgpu_kernel void @buffer_store_format_d16_xyz(<4 x i32> %rsrc, <4 x hal
;
; PACKED-LABEL: buffer_store_format_d16_xyz:
; PACKED: ; %bb.0: ; %main_body
-; PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-NEXT: s_and_b32 s5, s5, 0xffff
; PACKED-NEXT: v_mov_b32_e32 v0, s4
@@ -85,9 +81,7 @@ main_body:
define amdgpu_kernel void @buffer_store_format_d16_xyzw(<4 x i32> %rsrc, <4 x half> %data, i32 %voffset) {
; UNPACKED-LABEL: buffer_store_format_d16_xyzw:
; UNPACKED: ; %bb.0: ; %main_body
-; UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; UNPACKED-NEXT: s_lshr_b32 s7, s5, 16
; UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
@@ -103,9 +97,7 @@ define amdgpu_kernel void @buffer_store_format_d16_xyzw(<4 x i32> %rsrc, <4 x ha
;
; PACKED-LABEL: buffer_store_format_d16_xyzw:
; PACKED: ; %bb.0: ; %main_body
-; PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-NEXT: v_mov_b32_e32 v0, s4
; PACKED-NEXT: v_mov_b32_e32 v1, s5
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.format.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.format.d16.ll
index 1d2e325e98125..5d7b4c3c2e605 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.format.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.raw.ptr.buffer.store.format.d16.ll
@@ -50,9 +50,7 @@ main_body:
define amdgpu_kernel void @buffer_store_format_d16_xyz(ptr addrspace(8) %rsrc, <4 x half> %data, i32 %voffset) {
; UNPACKED-LABEL: buffer_store_format_d16_xyz:
; UNPACKED: ; %bb.0: ; %main_body
-; UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
; UNPACKED-NEXT: s_lshr_b32 s7, s4, 16
@@ -66,9 +64,7 @@ define amdgpu_kernel void @buffer_store_format_d16_xyz(ptr addrspace(8) %rsrc, <
;
; PACKED-LABEL: buffer_store_format_d16_xyz:
; PACKED: ; %bb.0: ; %main_body
-; PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-NEXT: s_and_b32 s5, s5, 0xffff
; PACKED-NEXT: v_mov_b32_e32 v0, s4
@@ -85,9 +81,7 @@ main_body:
define amdgpu_kernel void @buffer_store_format_d16_xyzw(ptr addrspace(8) %rsrc, <4 x half> %data, i32 %voffset) {
; UNPACKED-LABEL: buffer_store_format_d16_xyzw:
; UNPACKED: ; %bb.0: ; %main_body
-; UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; UNPACKED-NEXT: s_lshr_b32 s7, s5, 16
; UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
@@ -103,9 +97,7 @@ define amdgpu_kernel void @buffer_store_format_d16_xyzw(ptr addrspace(8) %rsrc,
;
; PACKED-LABEL: buffer_store_format_d16_xyzw:
; PACKED: ; %bb.0: ; %main_body
-; PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-NEXT: v_mov_b32_e32 v0, s4
; PACKED-NEXT: v_mov_b32_e32 v1, s5
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
index 81aa884f2829c..fd14c78349fe0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.add.ll
@@ -16,15 +16,14 @@
define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8DAGISEL-NEXT: s_mul_i32 s0, s2, s3
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX8DAGISEL-NEXT: flat_store_dword v[0:1], v2
; GFX8DAGISEL-NEXT: s_endpgm
;
@@ -44,13 +43,12 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: uniform_value:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
@@ -70,14 +68,12 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: uniform_value:
; GFX1064DAGISEL: ; %bb.0: ; %entry
-; GFX1064DAGISEL-NEXT: s_clause 0x1
-; GFX1064DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
@@ -98,13 +94,11 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: uniform_value:
; GFX1032DAGISEL: ; %bb.0: ; %entry
-; GFX1032DAGISEL-NEXT: s_clause 0x1
-; GFX1032DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -126,15 +120,13 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: uniform_value:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s6, s2
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
@@ -156,14 +148,13 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: uniform_value:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_mul_i32 s2, s2, s3
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
@@ -466,15 +457,14 @@ entry:
define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_dpp:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8DAGISEL-NEXT: s_mul_i32 s0, s2, s3
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX8DAGISEL-NEXT: flat_store_dword v[0:1], v2
; GFX8DAGISEL-NEXT: s_endpgm
;
@@ -494,13 +484,12 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX9DAGISEL-LABEL: uniform_value_dpp:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
@@ -520,14 +509,12 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1064DAGISEL-LABEL: uniform_value_dpp:
; GFX1064DAGISEL: ; %bb.0: ; %entry
-; GFX1064DAGISEL-NEXT: s_clause 0x1
-; GFX1064DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
@@ -548,13 +535,11 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1032DAGISEL-LABEL: uniform_value_dpp:
; GFX1032DAGISEL: ; %bb.0: ; %entry
-; GFX1032DAGISEL-NEXT: s_clause 0x1
-; GFX1032DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -576,15 +561,13 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1164DAGISEL-LABEL: uniform_value_dpp:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s6, s2
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
@@ -606,14 +589,13 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1132DAGISEL-LABEL: uniform_value_dpp:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_mul_i32 s2, s2, s3
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
@@ -2308,78 +2290,77 @@ entry:
define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: divergent_cfg:
; GFX8DAGISEL: ; %bb.0: ; %entry
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX8DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX8DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX8DAGISEL-NEXT: s_mul_i32 s6, s2, s3
+; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX8DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX8DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8DAGISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8DAGISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8DAGISEL-NEXT: s_add_i32 s6, s6, s8
-; GFX8DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8DAGISEL-NEXT: ; %bb.5:
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8DAGISEL-NEXT: flat_store_dword v[2:3], v1
; GFX8DAGISEL-NEXT: s_endpgm
;
; GFX8GISEL-LABEL: divergent_cfg:
; GFX8GISEL: ; %bb.0: ; %entry
+; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX8GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX8GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8GISEL-NEXT: ; %bb.1: ; %else
; GFX8GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX8GISEL-NEXT: ; implicit-def: $vgpr0
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX8GISEL-NEXT: s_mul_i32 s6, s6, s4
; GFX8GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8GISEL-NEXT: ; %bb.3: ; %if
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8GISEL-NEXT: s_mov_b32 s6, 0
; GFX8GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8GISEL-NEXT: s_add_i32 s6, s6, s8
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8GISEL-NEXT: ; %bb.5:
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8GISEL-NEXT: .LBB6_6: ; %endif
-; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8GISEL-NEXT: v_mov_b32_e32 v2, s0
@@ -2388,77 +2369,76 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: divergent_cfg:
; GFX9DAGISEL: ; %bb.0: ; %entry
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX9DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX9DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX9DAGISEL-NEXT: s_mul_i32 s6, s2, s3
+; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX9DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX9DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9DAGISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9DAGISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9DAGISEL-NEXT: s_add_i32 s6, s6, s8
-; GFX9DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9DAGISEL-NEXT: ; %bb.5:
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
;
; GFX9GISEL-LABEL: divergent_cfg:
; GFX9GISEL: ; %bb.0: ; %entry
+; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX9GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9GISEL-NEXT: ; %bb.1: ; %else
; GFX9GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9GISEL-NEXT: ; implicit-def: $vgpr0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX9GISEL-NEXT: s_mul_i32 s6, s6, s4
; GFX9GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9GISEL-NEXT: ; %bb.3: ; %if
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9GISEL-NEXT: s_mov_b32 s6, 0
; GFX9GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9GISEL-NEXT: s_add_i32 s6, s6, s8
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9GISEL-NEXT: ; %bb.5:
; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9GISEL-NEXT: .LBB6_6: ; %endif
-; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2466,77 +2446,76 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: divergent_cfg:
; GFX1064DAGISEL: ; %bb.0: ; %entry
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1064DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1064DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX1064DAGISEL-NEXT: s_mul_i32 s6, s2, s3
; GFX1064DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX1064DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064DAGISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064DAGISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064DAGISEL-NEXT: s_add_i32 s6, s6, s8
-; GFX1064DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064DAGISEL-NEXT: ; %bb.5:
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
;
; GFX1064GISEL-LABEL: divergent_cfg:
; GFX1064GISEL: ; %bb.0: ; %entry
+; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX1064GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064GISEL-NEXT: ; %bb.1: ; %else
; GFX1064GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1064GISEL-NEXT: s_mul_i32 s6, s6, s4
; GFX1064GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064GISEL-NEXT: ; %bb.3: ; %if
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: s_mov_b32 s6, 0
; GFX1064GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064GISEL-NEXT: s_add_i32 s6, s6, s8
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064GISEL-NEXT: ; %bb.5:
; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2544,77 +2523,76 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: divergent_cfg:
; GFX1032DAGISEL: ; %bb.0: ; %entry
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc_lo, 15, v0
-; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s4, vcc_lo
+; GFX1032DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1032DAGISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1032DAGISEL-NEXT: s_mul_i32 s3, s2, s3
; GFX1032DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX1032DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032DAGISEL-NEXT: s_mov_b32 s3, 0
; GFX1032DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032DAGISEL-NEXT: s_add_i32 s1, s1, s6
-; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032DAGISEL-NEXT: s_add_i32 s3, s3, s6
+; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032DAGISEL-NEXT: ; %bb.5:
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
; GFX1032GISEL-LABEL: divergent_cfg:
; GFX1032GISEL: ; %bb.0: ; %entry
+; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 16, v0
-; GFX1032GISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1032GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032GISEL-NEXT: ; %bb.1: ; %else
-; GFX1032GISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX1032GISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1032GISEL-NEXT: s_mul_i32 s3, s3, s4
; GFX1032GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032GISEL-NEXT: ; %bb.3: ; %if
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: s_mov_b32 s1, 0
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032GISEL-NEXT: s_mov_b32 s3, 0
; GFX1032GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032GISEL-NEXT: s_add_i32 s1, s1, s6
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032GISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032GISEL-NEXT: s_add_i32 s3, s3, s6
+; GFX1032GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032GISEL-NEXT: ; %bb.5:
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2622,85 +2600,84 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: divergent_cfg:
; GFX1164DAGISEL: ; %bb.0: ; %entry
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1164DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1164DAGISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1164DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_mul_i32 s6, s2, s3
; GFX1164DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX1164DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164DAGISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164DAGISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164DAGISEL-NEXT: s_add_i32 s6, s6, s8
-; GFX1164DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164DAGISEL-NEXT: ; %bb.5:
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
; GFX1164GISEL-LABEL: divergent_cfg:
; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1164GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1164GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164GISEL-NEXT: ; %bb.1: ; %else
; GFX1164GISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1164GISEL-NEXT: s_mul_i32 s6, s6, s4
; GFX1164GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164GISEL-NEXT: ; %bb.3: ; %if
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: s_mov_b32 s6, 0
; GFX1164GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164GISEL-NEXT: s_add_i32 s6, s6, s8
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164GISEL-NEXT: ; %bb.5:
; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2708,85 +2685,84 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: divergent_cfg:
; GFX1132DAGISEL: ; %bb.0: ; %entry
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1132DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1132DAGISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1132DAGISEL-NEXT: s_mul_i32 s3, s2, s3
; GFX1132DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, 0
; GFX1132DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132DAGISEL-NEXT: s_add_i32 s1, s1, s6
-; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132DAGISEL-NEXT: s_add_i32 s3, s3, s6
+; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132DAGISEL-NEXT: ; %bb.5:
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
; GFX1132GISEL-LABEL: divergent_cfg:
; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1132GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1132GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132GISEL-NEXT: ; %bb.1: ; %else
-; GFX1132GISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: s_load_b32 s3, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX1132GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1132GISEL-NEXT: s_mul_i32 s3, s3, s4
; GFX1132GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132GISEL-NEXT: ; %bb.3: ; %if
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s1, 0
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132GISEL-NEXT: s_mov_b32 s3, 0
; GFX1132GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132GISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132GISEL-NEXT: s_add_i32 s1, s1, s6
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132GISEL-NEXT: s_add_i32 s3, s3, s6
+; GFX1132GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132GISEL-NEXT: ; %bb.5:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2794,45 +2770,44 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX12DAGISEL-LABEL: divergent_cfg:
; GFX12DAGISEL: ; %bb.0: ; %entry
+; GFX12DAGISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX12DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX12DAGISEL-NEXT: ; implicit-def: $sgpr1
+; GFX12DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX12DAGISEL-NEXT: ; implicit-def: $sgpr3
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX12DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX12DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX12DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX12DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX12DAGISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
-; GFX12DAGISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX12DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX12DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX12DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12DAGISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX12DAGISEL-NEXT: s_mul_i32 s3, s2, s3
; GFX12DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX12DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX12DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX12DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX12DAGISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX12DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX12DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX12DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX12DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX12DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX12DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX12DAGISEL-NEXT: s_mov_b32 s3, 0
; GFX12DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX12DAGISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX12DAGISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX12DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX12DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX12DAGISEL-NEXT: s_add_co_i32 s1, s1, s6
+; GFX12DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX12DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX12DAGISEL-NEXT: s_add_co_i32 s3, s3, s6
; GFX12DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX12DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX12DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX12DAGISEL-NEXT: ; %bb.5:
-; GFX12DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX12DAGISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX12DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX12DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX12DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
; GFX12DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12DAGISEL-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
index e39f81a2f6267..b819cbe20636f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.and.ll
@@ -15,8 +15,7 @@
define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -37,8 +36,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: uniform_value:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -57,9 +55,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX10DAGISEL-LABEL: uniform_value:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -79,9 +75,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: uniform_value:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -101,9 +95,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: uniform_value:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -370,8 +362,7 @@ entry:
define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_dpp:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -392,8 +383,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX9DAGISEL-LABEL: uniform_value_dpp:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -412,9 +402,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX10DAGISEL-LABEL: uniform_value_dpp:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -434,9 +422,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1164DAGISEL-LABEL: uniform_value_dpp:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -456,9 +442,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1132DAGISEL-LABEL: uniform_value_dpp:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2000,18 +1984,17 @@ entry:
define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: divergent_cfg:
; GFX8DAGISEL: ; %bb.0: ; %entry
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX8DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8DAGISEL-NEXT: ; %bb.3: ; %if
; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2026,46 +2009,44 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-NEXT: ; %bb.5:
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8DAGISEL-NEXT: flat_store_dword v[2:3], v1
; GFX8DAGISEL-NEXT: s_endpgm
;
; GFX8GISEL-LABEL: divergent_cfg:
; GFX8GISEL: ; %bb.0: ; %entry
+; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX8GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX8GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8GISEL-NEXT: ; %bb.1: ; %else
-; GFX8GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX8GISEL-NEXT: ; implicit-def: $vgpr0
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_mov_b32 s2, s2
+; GFX8GISEL-NEXT: s_mov_b32 s6, s4
; GFX8GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8GISEL-NEXT: ; %bb.3: ; %if
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8GISEL-NEXT: s_mov_b32 s6, -1
; GFX8GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8GISEL-NEXT: s_and_b32 s6, s6, s8
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8GISEL-NEXT: ; %bb.5:
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8GISEL-NEXT: .LBB6_6: ; %endif
-; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8GISEL-NEXT: v_mov_b32_e32 v2, s0
@@ -2074,18 +2055,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: divergent_cfg:
; GFX9DAGISEL: ; %bb.0: ; %entry
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX9DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9DAGISEL-NEXT: ; %bb.3: ; %if
; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2100,45 +2080,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX9DAGISEL-NEXT: ; %bb.5:
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
;
; GFX9GISEL-LABEL: divergent_cfg:
; GFX9GISEL: ; %bb.0: ; %entry
+; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX9GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9GISEL-NEXT: ; %bb.1: ; %else
-; GFX9GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX9GISEL-NEXT: ; implicit-def: $vgpr0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_mov_b32 s2, s2
+; GFX9GISEL-NEXT: s_mov_b32 s6, s4
; GFX9GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9GISEL-NEXT: ; %bb.3: ; %if
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9GISEL-NEXT: s_mov_b32 s6, -1
; GFX9GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9GISEL-NEXT: s_and_b32 s6, s6, s8
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9GISEL-NEXT: ; %bb.5:
; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9GISEL-NEXT: .LBB6_6: ; %endif
-; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2146,18 +2124,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: divergent_cfg:
; GFX1064DAGISEL: ; %bb.0: ; %entry
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1064DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX1064DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2172,45 +2149,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1064DAGISEL-NEXT: ; %bb.5:
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
;
; GFX1064GISEL-LABEL: divergent_cfg:
; GFX1064GISEL: ; %bb.0: ; %entry
+; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX1064GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064GISEL-NEXT: ; %bb.1: ; %else
-; GFX1064GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX1064GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1064GISEL-NEXT: s_mov_b32 s6, s4
; GFX1064GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064GISEL-NEXT: ; %bb.3: ; %if
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: s_mov_b32 s6, -1
; GFX1064GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064GISEL-NEXT: s_and_b32 s6, s6, s8
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064GISEL-NEXT: ; %bb.5:
; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2218,71 +2193,68 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: divergent_cfg:
; GFX1032DAGISEL: ; %bb.0: ; %entry
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc_lo, 15, v0
-; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s3, vcc_lo
+; GFX1032DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1032DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1032DAGISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: s_mov_b32 s1, -1
+; GFX1032DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032DAGISEL-NEXT: s_mov_b32 s2, -1
; GFX1032DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032DAGISEL-NEXT: s_and_b32 s1, s1, s6
-; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032DAGISEL-NEXT: s_and_b32 s2, s2, s6
+; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032DAGISEL-NEXT: ; %bb.5:
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1032DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
; GFX1032GISEL-LABEL: divergent_cfg:
; GFX1032GISEL: ; %bb.0: ; %entry
+; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 16, v0
-; GFX1032GISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1032GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032GISEL-NEXT: ; %bb.1: ; %else
-; GFX1032GISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX1032GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1032GISEL-NEXT: s_mov_b32 s3, s3
; GFX1032GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032GISEL-NEXT: ; %bb.3: ; %if
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: s_mov_b32 s1, -1
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032GISEL-NEXT: s_mov_b32 s3, -1
; GFX1032GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032GISEL-NEXT: s_and_b32 s1, s1, s6
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032GISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032GISEL-NEXT: s_and_b32 s3, s3, s6
+; GFX1032GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032GISEL-NEXT: ; %bb.5:
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2290,20 +2262,20 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: divergent_cfg:
; GFX1164DAGISEL: ; %bb.0: ; %entry
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1164DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1164DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2319,48 +2291,46 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1164DAGISEL-NEXT: ; %bb.5:
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
; GFX1164GISEL-LABEL: divergent_cfg:
; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1164GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1164GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164GISEL-NEXT: ; %bb.1: ; %else
-; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b32 s4, s[4:5], 0x2c
; GFX1164GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1164GISEL-NEXT: s_mov_b32 s6, s4
; GFX1164GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164GISEL-NEXT: ; %bb.3: ; %if
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: s_mov_b32 s6, -1
; GFX1164GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164GISEL-NEXT: s_and_b32 s6, s6, s8
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164GISEL-NEXT: ; %bb.5:
; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2368,77 +2338,75 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: divergent_cfg:
; GFX1132DAGISEL: ; %bb.0: ; %entry
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1132DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1132DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1132DAGISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
; GFX1132DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: s_mov_b32 s1, -1
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: s_mov_b32 s2, -1
; GFX1132DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132DAGISEL-NEXT: s_and_b32 s1, s1, s6
-; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132DAGISEL-NEXT: s_and_b32 s2, s2, s6
+; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132DAGISEL-NEXT: ; %bb.5:
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1132DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
; GFX1132GISEL-LABEL: divergent_cfg:
; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1132GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1132GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132GISEL-NEXT: ; %bb.1: ; %else
-; GFX1132GISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b32 s3, s[4:5], 0x2c
; GFX1132GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1132GISEL-NEXT: s_mov_b32 s3, s3
; GFX1132GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132GISEL-NEXT: ; %bb.3: ; %if
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s1, -1
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132GISEL-NEXT: s_mov_b32 s3, -1
; GFX1132GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132GISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132GISEL-NEXT: s_and_b32 s1, s1, s6
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132GISEL-NEXT: s_and_b32 s3, s3, s6
+; GFX1132GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132GISEL-NEXT: ; %bb.5:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
index 2f95542150a83..152e7ecd16998 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fadd.ll
@@ -17,17 +17,16 @@
define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_float:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX8DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
-; GFX8DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX8DAGISEL-NEXT: v_cvt_f32_i32_e32 v2, s3
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX8DAGISEL-NEXT: v_mul_f32_e32 v2, s2, v2
+; GFX8DAGISEL-NEXT: v_readfirstlane_b32 s0, v2
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX8DAGISEL-NEXT: flat_store_dword v[0:1], v2
; GFX8DAGISEL-NEXT: s_endpgm
;
@@ -49,13 +48,12 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX9DAGISEL-LABEL: uniform_value_float:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX9DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX9DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, s2
@@ -79,12 +77,11 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1064DAGISEL-LABEL: uniform_value_float:
; GFX1064DAGISEL: ; %bb.0: ; %entry
-; GFX1064DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1064DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
; GFX1064DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX1064DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -109,12 +106,11 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1032DAGISEL-LABEL: uniform_value_float:
; GFX1032DAGISEL: ; %bb.0: ; %entry
-; GFX1032DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
; GFX1032DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX1032DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -139,13 +135,12 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1164DAGISEL-LABEL: uniform_value_float:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
; GFX1164DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
@@ -173,16 +168,16 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1132DAGISEL-LABEL: uniform_value_float:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX1132DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
@@ -1831,74 +1826,70 @@ entry:
define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in, float %in2) #0 {
; GFX8DAGISEL-LABEL: divergent_cfg_float:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr6
-; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX8DAGISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX8DAGISEL-NEXT: ; %bb.1: ; %else
; GFX8DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX8DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mul_f32_e32 v0, s0, v0
+; GFX8DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX8DAGISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX8DAGISEL-NEXT: .LBB4_2: ; %Flow
-; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX8DAGISEL-NEXT: ; %bb.3: ; %if
; GFX8DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX8DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX8DAGISEL-NEXT: v_mul_f32_e32 v0, s1, v0
-; GFX8DAGISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX8DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX8DAGISEL-NEXT: v_mul_f32_e32 v0, s3, v0
+; GFX8DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX8DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s1
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s1
; GFX8DAGISEL-NEXT: flat_store_dword v[1:2], v0
; GFX8DAGISEL-NEXT: s_endpgm
;
; GFX8GISEL-LABEL: divergent_cfg_float:
; GFX8GISEL: ; %bb.0: ; %entry
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
; GFX8GISEL-NEXT: ; implicit-def: $sgpr6
-; GFX8GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX8GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX8GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX8GISEL-NEXT: ; %bb.1: ; %else
; GFX8GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX8GISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: v_mul_f32_e32 v0, s0, v0
+; GFX8GISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX8GISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX8GISEL-NEXT: .LBB4_2: ; %Flow
-; GFX8GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8GISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX8GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX8GISEL-NEXT: ; %bb.3: ; %if
; GFX8GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX8GISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX8GISEL-NEXT: v_mul_f32_e32 v0, s1, v0
-; GFX8GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX8GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX8GISEL-NEXT: v_mul_f32_e32 v0, s3, v0
+; GFX8GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX8GISEL-NEXT: .LBB4_4: ; %endif
-; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_mov_b32_e32 v2, s1
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s0
@@ -1907,36 +1898,34 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX9DAGISEL-LABEL: divergent_cfg_float:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr6
-; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX9DAGISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX9DAGISEL-NEXT: ; %bb.1: ; %else
; GFX9DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: v_mul_f32_e32 v0, s0, v0
+; GFX9DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX9DAGISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX9DAGISEL-NEXT: .LBB4_2: ; %Flow
-; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX9DAGISEL-NEXT: ; %bb.3: ; %if
; GFX9DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX9DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9DAGISEL-NEXT: v_mul_f32_e32 v0, s1, v0
-; GFX9DAGISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX9DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX9DAGISEL-NEXT: v_mul_f32_e32 v0, s3, v0
+; GFX9DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX9DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: global_store_dword v1, v0, s[0:1]
@@ -1944,36 +1933,34 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX9GISEL-LABEL: divergent_cfg_float:
; GFX9GISEL: ; %bb.0: ; %entry
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
; GFX9GISEL-NEXT: ; implicit-def: $sgpr6
-; GFX9GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX9GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX9GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX9GISEL-NEXT: ; %bb.1: ; %else
; GFX9GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9GISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: v_mul_f32_e32 v0, s0, v0
+; GFX9GISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX9GISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX9GISEL-NEXT: .LBB4_2: ; %Flow
-; GFX9GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9GISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX9GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX9GISEL-NEXT: ; %bb.3: ; %if
; GFX9GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX9GISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9GISEL-NEXT: v_mul_f32_e32 v0, s1, v0
-; GFX9GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX9GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX9GISEL-NEXT: v_mul_f32_e32 v0, s3, v0
+; GFX9GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX9GISEL-NEXT: .LBB4_4: ; %endif
-; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9GISEL-NEXT: global_store_dword v1, v0, s[0:1]
@@ -1981,36 +1968,34 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX1064DAGISEL-LABEL: divergent_cfg_float:
; GFX1064DAGISEL: ; %bb.0: ; %entry
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr6
-; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX1064DAGISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1064DAGISEL-NEXT: ; %bb.1: ; %else
; GFX1064DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX1064DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: v_mul_f32_e32 v0, s0, v0
+; GFX1064DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX1064DAGISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064DAGISEL-NEXT: .LBB4_2: ; %Flow
-; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1064DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1064DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX1064DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX1064DAGISEL-NEXT: v_mul_f32_e32 v0, s1, v0
-; GFX1064DAGISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX1064DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1064DAGISEL-NEXT: v_mul_f32_e32 v0, s3, v0
+; GFX1064DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1064DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: global_store_dword v1, v0, s[0:1]
@@ -2018,36 +2003,34 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX1064GISEL-LABEL: divergent_cfg_float:
; GFX1064GISEL: ; %bb.0: ; %entry
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX1064GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
; GFX1064GISEL-NEXT: ; implicit-def: $sgpr6
-; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX1064GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1064GISEL-NEXT: ; %bb.1: ; %else
; GFX1064GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX1064GISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: v_mul_f32_e32 v0, s0, v0
+; GFX1064GISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX1064GISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064GISEL-NEXT: .LBB4_2: ; %Flow
-; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1064GISEL-NEXT: ; %bb.3: ; %if
; GFX1064GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX1064GISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX1064GISEL-NEXT: v_mul_f32_e32 v0, s1, v0
-; GFX1064GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX1064GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1064GISEL-NEXT: v_mul_f32_e32 v0, s3, v0
+; GFX1064GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1064GISEL-NEXT: .LBB4_4: ; %endif
-; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064GISEL-NEXT: global_store_dword v1, v0, s[0:1]
@@ -2055,85 +2038,81 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX1032DAGISEL-LABEL: divergent_cfg_float:
; GFX1032DAGISEL: ; %bb.0: ; %entry
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc_lo, 15, v0
-; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr3
-; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
-; GFX1032DAGISEL-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr5
+; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s4, vcc_lo
+; GFX1032DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1032DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1032DAGISEL-NEXT: s_mov_b32 s5, exec_lo
+; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s5, s5
+; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: v_mul_f32_e32 v0, s0, v0
-; GFX1032DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1032DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX1032DAGISEL-NEXT: v_readfirstlane_b32 s5, v0
; GFX1032DAGISEL-NEXT: .LBB4_2: ; %Flow
; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s0, s2
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1032DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX1032DAGISEL-NEXT: v_mul_f32_e32 v0, s1, v0
-; GFX1032DAGISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX1032DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s4, s4
+; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s4
+; GFX1032DAGISEL-NEXT: v_mul_f32_e32 v0, s3, v0
+; GFX1032DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX1032DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
; GFX1032GISEL-LABEL: divergent_cfg_float:
; GFX1032GISEL: ; %bb.0: ; %entry
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 16, v0
-; GFX1032GISEL-NEXT: ; implicit-def: $sgpr3
-; GFX1032GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
-; GFX1032GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX1032GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX1032GISEL-NEXT: s_and_saveexec_b32 s4, vcc_lo
+; GFX1032GISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1032GISEL-NEXT: ; %bb.1: ; %else
-; GFX1032GISEL-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032GISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1032GISEL-NEXT: s_mov_b32 s5, exec_lo
+; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s5, s5
+; GFX1032GISEL-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: v_mul_f32_e32 v0, s0, v0
-; GFX1032GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1032GISEL-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX1032GISEL-NEXT: v_readfirstlane_b32 s5, v0
; GFX1032GISEL-NEXT: .LBB4_2: ; %Flow
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: s_or_saveexec_b32 s0, s2
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032GISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1032GISEL-NEXT: ; %bb.3: ; %if
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1032GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX1032GISEL-NEXT: v_mul_f32_e32 v0, s1, v0
-; GFX1032GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s4, s4
+; GFX1032GISEL-NEXT: v_cvt_f32_i32_e32 v0, s4
+; GFX1032GISEL-NEXT: v_mul_f32_e32 v0, s3, v0
+; GFX1032GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX1032GISEL-NEXT: .LBB4_4: ; %endif
-; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032GISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX1032GISEL-NEXT: s_endpgm
;
; GFX1164DAGISEL-LABEL: divergent_cfg_float:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1164DAGISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX1164DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1164DAGISEL-NEXT: ; %bb.1: ; %else
; GFX1164DAGISEL-NEXT: s_mov_b64 s[6:7], exec
@@ -2142,28 +2121,26 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
; GFX1164DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164DAGISEL-NEXT: v_mul_f32_e32 v0, s0, v0
+; GFX1164DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX1164DAGISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX1164DAGISEL-NEXT: .LBB4_2: ; %Flow
-; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1164DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1164DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX1164DAGISEL-NEXT: v_mul_f32_e32 v0, s1, v0
+; GFX1164DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1164DAGISEL-NEXT: v_mul_f32_e32 v0, s3, v0
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164DAGISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1164DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1164DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: global_store_b32 v1, v0, s[0:1]
@@ -2171,13 +2148,13 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX1164GISEL-LABEL: divergent_cfg_float:
; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1164GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX1164GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1164GISEL-NEXT: ; %bb.1: ; %else
; GFX1164GISEL-NEXT: s_mov_b64 s[6:7], exec
@@ -2186,28 +2163,26 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
; GFX1164GISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164GISEL-NEXT: v_mul_f32_e32 v0, s0, v0
+; GFX1164GISEL-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX1164GISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX1164GISEL-NEXT: .LBB4_2: ; %Flow
-; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1164GISEL-NEXT: ; %bb.3: ; %if
; GFX1164GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
+; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164GISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX1164GISEL-NEXT: v_mul_f32_e32 v0, s1, v0
+; GFX1164GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1164GISEL-NEXT: v_mul_f32_e32 v0, s3, v0
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1164GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1164GISEL-NEXT: .LBB4_4: ; %endif
-; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
@@ -2215,136 +2190,130 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX1132DAGISEL-LABEL: divergent_cfg_float:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr5
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1132DAGISEL-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX1132DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1132DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132DAGISEL-NEXT: s_mov_b32 s5, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s5, s5
+; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132DAGISEL-NEXT: v_mul_f32_e32 v0, s0, v0
-; GFX1132DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1132DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX1132DAGISEL-NEXT: v_readfirstlane_b32 s5, v0
; GFX1132DAGISEL-NEXT: .LBB4_2: ; %Flow
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s0, s2
+; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1132DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s4, s4
+; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132DAGISEL-NEXT: v_mul_f32_e32 v0, s1, v0
-; GFX1132DAGISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX1132DAGISEL-NEXT: v_mul_f32_e32 v0, s3, v0
+; GFX1132DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX1132DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
; GFX1132GISEL-LABEL: divergent_cfg_float:
; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132GISEL-NEXT: ; implicit-def: $sgpr5
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1132GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX1132GISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1132GISEL-NEXT: ; %bb.1: ; %else
-; GFX1132GISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132GISEL-NEXT: s_mov_b32 s5, exec_lo
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132GISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s5, s5
+; GFX1132GISEL-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132GISEL-NEXT: v_mul_f32_e32 v0, s0, v0
-; GFX1132GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1132GISEL-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX1132GISEL-NEXT: v_readfirstlane_b32 s5, v0
; GFX1132GISEL-NEXT: .LBB4_2: ; %Flow
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: s_or_saveexec_b32 s0, s2
+; GFX1132GISEL-NEXT: s_or_saveexec_b32 s2, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1132GISEL-NEXT: ; %bb.3: ; %if
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1132GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s4, s4
+; GFX1132GISEL-NEXT: v_cvt_f32_i32_e32 v0, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132GISEL-NEXT: v_mul_f32_e32 v0, s1, v0
-; GFX1132GISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX1132GISEL-NEXT: v_mul_f32_e32 v0, s3, v0
+; GFX1132GISEL-NEXT: v_readfirstlane_b32 s3, v0
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX1132GISEL-NEXT: .LBB4_4: ; %endif
-; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1132GISEL-NEXT: s_endpgm
;
; GFX12DAGISEL-LABEL: divergent_cfg_float:
; GFX12DAGISEL: ; %bb.0: ; %entry
-; GFX12DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX12DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX12DAGISEL-NEXT: ; implicit-def: $sgpr3
+; GFX12DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX12DAGISEL-NEXT: ; implicit-def: $sgpr5
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX12DAGISEL-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX12DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX12DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX12DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX12DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX12DAGISEL-NEXT: s_mov_b32 s5, exec_lo
; GFX12DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX12DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s5, s5
+; GFX12DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12DAGISEL-NEXT: v_mul_f32_e32 v0, s0, v0
-; GFX12DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX12DAGISEL-NEXT: v_mul_f32_e32 v0, s2, v0
+; GFX12DAGISEL-NEXT: v_readfirstlane_b32 s5, v0
; GFX12DAGISEL-NEXT: .LBB4_2: ; %Flow
; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12DAGISEL-NEXT: s_or_saveexec_b32 s0, s2
+; GFX12DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12DAGISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX12DAGISEL-NEXT: v_mov_b32_e32 v0, s5
; GFX12DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX12DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX12DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX12DAGISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX12DAGISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX12DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX12DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX12DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s4
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12DAGISEL-NEXT: v_mul_f32_e32 v0, s1, v0
-; GFX12DAGISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX12DAGISEL-NEXT: v_mul_f32_e32 v0, s3, v0
+; GFX12DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
; GFX12DAGISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12DAGISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX12DAGISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX12DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX12DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX12DAGISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
; GFX12DAGISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12DAGISEL-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
index 4e7a6ac5891fb..1c3862f1ff5d6 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmax.ll
@@ -17,8 +17,7 @@
define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_float:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -39,8 +38,7 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX9DAGISEL-LABEL: uniform_value_float:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -59,9 +57,7 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX10DAGISEL-LABEL: uniform_value_float:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -81,9 +77,7 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1164DAGISEL-LABEL: uniform_value_float:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -103,9 +97,7 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1132DAGISEL-LABEL: uniform_value_float:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
index bbe672f428741..52bdb7528e541 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fmin.ll
@@ -17,8 +17,7 @@
define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_float:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -39,8 +38,7 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX9DAGISEL-LABEL: uniform_value_float:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -59,9 +57,7 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX10DAGISEL-LABEL: uniform_value_float:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -81,9 +77,7 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1164DAGISEL-LABEL: uniform_value_float:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -103,9 +97,7 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1132DAGISEL-LABEL: uniform_value_float:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
index 22ff4f6063177..d1617700c03d3 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.fsub.ll
@@ -17,17 +17,16 @@
define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_float:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX8DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
-; GFX8DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX8DAGISEL-NEXT: v_cvt_f32_i32_e32 v2, s3
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX8DAGISEL-NEXT: v_mul_f32_e64 v2, -s2, v2
+; GFX8DAGISEL-NEXT: v_readfirstlane_b32 s0, v2
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX8DAGISEL-NEXT: flat_store_dword v[0:1], v2
; GFX8DAGISEL-NEXT: s_endpgm
;
@@ -49,13 +48,12 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX9DAGISEL-LABEL: uniform_value_float:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX9DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, 0
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX9DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX9DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, s2
@@ -79,12 +77,11 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1064DAGISEL-LABEL: uniform_value_float:
; GFX1064DAGISEL: ; %bb.0: ; %entry
-; GFX1064DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1064DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
; GFX1064DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX1064DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -109,12 +106,11 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1032DAGISEL-LABEL: uniform_value_float:
; GFX1032DAGISEL: ; %bb.0: ; %entry
-; GFX1032DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
; GFX1032DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX1032DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
@@ -139,13 +135,12 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1164DAGISEL-LABEL: uniform_value_float:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[0:1]
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1164DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
; GFX1164DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1164DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
@@ -173,16 +168,16 @@ define amdgpu_kernel void @uniform_value_float(ptr addrspace(1) %out, float %in)
;
; GFX1132DAGISEL-LABEL: uniform_value_float:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1132DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX1132DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
@@ -1864,74 +1859,70 @@ entry:
define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in, float %in2) #0 {
; GFX8DAGISEL-LABEL: divergent_cfg_float:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr6
-; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX8DAGISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX8DAGISEL-NEXT: ; %bb.1: ; %else
; GFX8DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX8DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
+; GFX8DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX8DAGISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX8DAGISEL-NEXT: .LBB4_2: ; %Flow
-; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX8DAGISEL-NEXT: ; %bb.3: ; %if
; GFX8DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX8DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX8DAGISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
-; GFX8DAGISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX8DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX8DAGISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
+; GFX8DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX8DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s1
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s1
; GFX8DAGISEL-NEXT: flat_store_dword v[1:2], v0
; GFX8DAGISEL-NEXT: s_endpgm
;
; GFX8GISEL-LABEL: divergent_cfg_float:
; GFX8GISEL: ; %bb.0: ; %entry
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
; GFX8GISEL-NEXT: ; implicit-def: $sgpr6
-; GFX8GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX8GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX8GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX8GISEL-NEXT: ; %bb.1: ; %else
; GFX8GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX8GISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
+; GFX8GISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX8GISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX8GISEL-NEXT: .LBB4_2: ; %Flow
-; GFX8GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8GISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX8GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX8GISEL-NEXT: ; %bb.3: ; %if
; GFX8GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX8GISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX8GISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
-; GFX8GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX8GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX8GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX8GISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
+; GFX8GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX8GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX8GISEL-NEXT: .LBB4_4: ; %endif
-; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_mov_b32_e32 v2, s1
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s0
@@ -1940,36 +1931,34 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX9DAGISEL-LABEL: divergent_cfg_float:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr6
-; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX9DAGISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX9DAGISEL-NEXT: ; %bb.1: ; %else
; GFX9DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
+; GFX9DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX9DAGISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX9DAGISEL-NEXT: .LBB4_2: ; %Flow
-; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX9DAGISEL-NEXT: ; %bb.3: ; %if
; GFX9DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX9DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9DAGISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
-; GFX9DAGISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX9DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX9DAGISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
+; GFX9DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX9DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: global_store_dword v1, v0, s[0:1]
@@ -1977,36 +1966,34 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX9GISEL-LABEL: divergent_cfg_float:
; GFX9GISEL: ; %bb.0: ; %entry
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
; GFX9GISEL-NEXT: ; implicit-def: $sgpr6
-; GFX9GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX9GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX9GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX9GISEL-NEXT: ; %bb.1: ; %else
; GFX9GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX9GISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
+; GFX9GISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX9GISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX9GISEL-NEXT: .LBB4_2: ; %Flow
-; GFX9GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9GISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX9GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX9GISEL-NEXT: ; %bb.3: ; %if
; GFX9GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX9GISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX9GISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
-; GFX9GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX9GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX9GISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
+; GFX9GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX9GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX9GISEL-NEXT: .LBB4_4: ; %endif
-; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9GISEL-NEXT: global_store_dword v1, v0, s[0:1]
@@ -2014,36 +2001,34 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX1064DAGISEL-LABEL: divergent_cfg_float:
; GFX1064DAGISEL: ; %bb.0: ; %entry
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr6
-; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX1064DAGISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1064DAGISEL-NEXT: ; %bb.1: ; %else
; GFX1064DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX1064DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
+; GFX1064DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX1064DAGISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064DAGISEL-NEXT: .LBB4_2: ; %Flow
-; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1064DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1064DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX1064DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX1064DAGISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
-; GFX1064DAGISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX1064DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1064DAGISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
+; GFX1064DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1064DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: global_store_dword v1, v0, s[0:1]
@@ -2051,36 +2036,34 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX1064GISEL-LABEL: divergent_cfg_float:
; GFX1064GISEL: ; %bb.0: ; %entry
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX1064GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
; GFX1064GISEL-NEXT: ; implicit-def: $sgpr6
-; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
-; GFX1064GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1064GISEL-NEXT: ; %bb.1: ; %else
; GFX1064GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s6, s[6:7]
; GFX1064GISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
+; GFX1064GISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX1064GISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX1064GISEL-NEXT: .LBB4_2: ; %Flow
-; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1064GISEL-NEXT: ; %bb.3: ; %if
; GFX1064GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
-; GFX1064GISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX1064GISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
-; GFX1064GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
+; GFX1064GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1064GISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
+; GFX1064GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1064GISEL-NEXT: .LBB4_4: ; %endif
-; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064GISEL-NEXT: global_store_dword v1, v0, s[0:1]
@@ -2088,85 +2071,81 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX1032DAGISEL-LABEL: divergent_cfg_float:
; GFX1032DAGISEL: ; %bb.0: ; %entry
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc_lo, 15, v0
-; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr3
-; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
-; GFX1032DAGISEL-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr5
+; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s4, vcc_lo
+; GFX1032DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1032DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1032DAGISEL-NEXT: s_mov_b32 s5, exec_lo
+; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s5, s5
+; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
-; GFX1032DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1032DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
+; GFX1032DAGISEL-NEXT: v_readfirstlane_b32 s5, v0
; GFX1032DAGISEL-NEXT: .LBB4_2: ; %Flow
; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s0, s2
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1032DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX1032DAGISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
-; GFX1032DAGISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX1032DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s4, s4
+; GFX1032DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s4
+; GFX1032DAGISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
+; GFX1032DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX1032DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
; GFX1032GISEL-LABEL: divergent_cfg_float:
; GFX1032GISEL: ; %bb.0: ; %entry
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 16, v0
-; GFX1032GISEL-NEXT: ; implicit-def: $sgpr3
-; GFX1032GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
-; GFX1032GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX1032GISEL-NEXT: ; implicit-def: $sgpr5
+; GFX1032GISEL-NEXT: s_and_saveexec_b32 s4, vcc_lo
+; GFX1032GISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1032GISEL-NEXT: ; %bb.1: ; %else
-; GFX1032GISEL-NEXT: s_mov_b32 s3, exec_lo
-; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032GISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1032GISEL-NEXT: s_mov_b32 s5, exec_lo
+; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s5, s5
+; GFX1032GISEL-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
-; GFX1032GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1032GISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
+; GFX1032GISEL-NEXT: v_readfirstlane_b32 s5, v0
; GFX1032GISEL-NEXT: .LBB4_2: ; %Flow
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: s_or_saveexec_b32 s0, s2
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032GISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1032GISEL-NEXT: ; %bb.3: ; %if
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1032GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
-; GFX1032GISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
-; GFX1032GISEL-NEXT: v_readfirstlane_b32 s1, v0
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s4, s4
+; GFX1032GISEL-NEXT: v_cvt_f32_i32_e32 v0, s4
+; GFX1032GISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
+; GFX1032GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX1032GISEL-NEXT: .LBB4_4: ; %endif
-; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032GISEL-NEXT: global_store_dword v1, v0, s[0:1]
; GFX1032GISEL-NEXT: s_endpgm
;
; GFX1164DAGISEL-LABEL: divergent_cfg_float:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1164DAGISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX1164DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1164DAGISEL-NEXT: ; %bb.1: ; %else
; GFX1164DAGISEL-NEXT: s_mov_b64 s[6:7], exec
@@ -2175,28 +2154,26 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
; GFX1164DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164DAGISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
+; GFX1164DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX1164DAGISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX1164DAGISEL-NEXT: .LBB4_2: ; %Flow
-; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1164DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1164DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX1164DAGISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
+; GFX1164DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1164DAGISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164DAGISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1164DAGISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1164DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: global_store_b32 v1, v0, s[0:1]
@@ -2204,13 +2181,13 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX1164GISEL-LABEL: divergent_cfg_float:
; GFX1164GISEL: ; %bb.0: ; %entry
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1164GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
+; GFX1164GISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1164GISEL-NEXT: ; %bb.1: ; %else
; GFX1164GISEL-NEXT: s_mov_b64 s[6:7], exec
@@ -2219,28 +2196,26 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
; GFX1164GISEL-NEXT: v_cvt_f32_i32_e32 v0, s6
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164GISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
+; GFX1164GISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
; GFX1164GISEL-NEXT: v_readfirstlane_b32 s6, v0
; GFX1164GISEL-NEXT: .LBB4_2: ; %Flow
-; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s6
-; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
+; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1164GISEL-NEXT: ; %bb.3: ; %if
; GFX1164GISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s0, s[6:7]
+; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s2, s[6:7]
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164GISEL-NEXT: v_cvt_f32_i32_e32 v0, s0
-; GFX1164GISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
+; GFX1164GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1164GISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1164GISEL-NEXT: v_readfirstlane_b32 s0, v0
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX1164GISEL-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, s2
; GFX1164GISEL-NEXT: .LBB4_4: ; %endif
-; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
@@ -2248,136 +2223,130 @@ define amdgpu_kernel void @divergent_cfg_float(ptr addrspace(1) %out, float %in,
;
; GFX1132DAGISEL-LABEL: divergent_cfg_float:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr5
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1132DAGISEL-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX1132DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1132DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132DAGISEL-NEXT: s_mov_b32 s5, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s5, s5
+; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132DAGISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
-; GFX1132DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1132DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
+; GFX1132DAGISEL-NEXT: v_readfirstlane_b32 s5, v0
; GFX1132DAGISEL-NEXT: .LBB4_2: ; %Flow
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s0, s2
+; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1132DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s4, s4
+; GFX1132DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132DAGISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
-; GFX1132DAGISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX1132DAGISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
+; GFX1132DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX1132DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
; GFX1132GISEL-LABEL: divergent_cfg_float:
; GFX1132GISEL: ; %bb.0: ; %entry
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132GISEL-NEXT: ; implicit-def: $sgpr5
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1132GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX1132GISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX1132GISEL-NEXT: ; %bb.1: ; %else
-; GFX1132GISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132GISEL-NEXT: s_mov_b32 s5, exec_lo
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132GISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s5, s5
+; GFX1132GISEL-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132GISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
-; GFX1132GISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1132GISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
+; GFX1132GISEL-NEXT: v_readfirstlane_b32 s5, v0
; GFX1132GISEL-NEXT: .LBB4_2: ; %Flow
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: s_or_saveexec_b32 s0, s2
+; GFX1132GISEL-NEXT: s_or_saveexec_b32 s2, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s3
-; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s5
+; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX1132GISEL-NEXT: ; %bb.3: ; %if
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1132GISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s4, s4
+; GFX1132GISEL-NEXT: v_cvt_f32_i32_e32 v0, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1132GISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
-; GFX1132GISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX1132GISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
+; GFX1132GISEL-NEXT: v_readfirstlane_b32 s3, v0
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX1132GISEL-NEXT: .LBB4_4: ; %endif
-; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1132GISEL-NEXT: s_endpgm
;
; GFX12DAGISEL-LABEL: divergent_cfg_float:
; GFX12DAGISEL: ; %bb.0: ; %entry
-; GFX12DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x2c
+; GFX12DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX12DAGISEL-NEXT: ; implicit-def: $sgpr3
+; GFX12DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX12DAGISEL-NEXT: ; implicit-def: $sgpr5
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX12DAGISEL-NEXT: s_xor_b32 s2, exec_lo, s2
+; GFX12DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX12DAGISEL-NEXT: s_cbranch_execz .LBB4_2
; GFX12DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX12DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX12DAGISEL-NEXT: s_mov_b32 s5, exec_lo
; GFX12DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX12DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s3
+; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s5, s5
+; GFX12DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s5
; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12DAGISEL-NEXT: v_mul_f32_e64 v0, -s0, v0
-; GFX12DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
+; GFX12DAGISEL-NEXT: v_mul_f32_e64 v0, -s2, v0
+; GFX12DAGISEL-NEXT: v_readfirstlane_b32 s5, v0
; GFX12DAGISEL-NEXT: .LBB4_2: ; %Flow
; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12DAGISEL-NEXT: s_or_saveexec_b32 s0, s2
+; GFX12DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12DAGISEL-NEXT: v_mov_b32_e32 v0, s3
+; GFX12DAGISEL-NEXT: v_mov_b32_e32 v0, s5
; GFX12DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX12DAGISEL-NEXT: s_cbranch_execz .LBB4_4
; GFX12DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX12DAGISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX12DAGISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX12DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX12DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s2
+; GFX12DAGISEL-NEXT: v_cvt_f32_i32_e32 v0, s4
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12DAGISEL-NEXT: v_mul_f32_e64 v0, -s1, v0
-; GFX12DAGISEL-NEXT: v_readfirstlane_b32 s1, v0
+; GFX12DAGISEL-NEXT: v_mul_f32_e64 v0, -s3, v0
+; GFX12DAGISEL-NEXT: v_readfirstlane_b32 s3, v0
; GFX12DAGISEL-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12DAGISEL-NEXT: v_mov_b32_e32 v0, s1
+; GFX12DAGISEL-NEXT: v_mov_b32_e32 v0, s3
; GFX12DAGISEL-NEXT: .LBB4_4: ; %endif
-; GFX12DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX12DAGISEL-NEXT: v_mov_b32_e32 v1, 0
-; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
; GFX12DAGISEL-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12DAGISEL-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
index bd2cf4280bdf8..45b3cbf99efcf 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.max.ll
@@ -15,8 +15,7 @@
define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -37,8 +36,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: uniform_value:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -57,9 +55,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX10DAGISEL-LABEL: uniform_value:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -79,9 +75,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: uniform_value:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -101,9 +95,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: uniform_value:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -370,8 +362,7 @@ entry:
define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_dpp:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -392,8 +383,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX9DAGISEL-LABEL: uniform_value_dpp:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -412,9 +402,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX10DAGISEL-LABEL: uniform_value_dpp:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -434,9 +422,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1164DAGISEL-LABEL: uniform_value_dpp:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -456,9 +442,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1132DAGISEL-LABEL: uniform_value_dpp:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2132,18 +2116,17 @@ entry:
define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: divergent_cfg:
; GFX8DAGISEL: ; %bb.0: ; %entry
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX8DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8DAGISEL-NEXT: ; %bb.3: ; %if
; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2158,46 +2141,44 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-NEXT: ; %bb.5:
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8DAGISEL-NEXT: flat_store_dword v[2:3], v1
; GFX8DAGISEL-NEXT: s_endpgm
;
; GFX8GISEL-LABEL: divergent_cfg:
; GFX8GISEL: ; %bb.0: ; %entry
+; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX8GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX8GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8GISEL-NEXT: ; %bb.1: ; %else
-; GFX8GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX8GISEL-NEXT: ; implicit-def: $vgpr0
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_mov_b32 s2, s2
+; GFX8GISEL-NEXT: s_mov_b32 s6, s4
; GFX8GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8GISEL-NEXT: ; %bb.3: ; %if
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8GISEL-NEXT: s_brev_b32 s6, 1
; GFX8GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8GISEL-NEXT: s_max_i32 s6, s6, s8
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8GISEL-NEXT: ; %bb.5:
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8GISEL-NEXT: .LBB6_6: ; %endif
-; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8GISEL-NEXT: v_mov_b32_e32 v2, s0
@@ -2206,18 +2187,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: divergent_cfg:
; GFX9DAGISEL: ; %bb.0: ; %entry
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX9DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9DAGISEL-NEXT: ; %bb.3: ; %if
; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2232,45 +2212,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX9DAGISEL-NEXT: ; %bb.5:
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
;
; GFX9GISEL-LABEL: divergent_cfg:
; GFX9GISEL: ; %bb.0: ; %entry
+; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX9GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9GISEL-NEXT: ; %bb.1: ; %else
-; GFX9GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX9GISEL-NEXT: ; implicit-def: $vgpr0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_mov_b32 s2, s2
+; GFX9GISEL-NEXT: s_mov_b32 s6, s4
; GFX9GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9GISEL-NEXT: ; %bb.3: ; %if
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9GISEL-NEXT: s_brev_b32 s6, 1
; GFX9GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9GISEL-NEXT: s_max_i32 s6, s6, s8
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9GISEL-NEXT: ; %bb.5:
; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9GISEL-NEXT: .LBB6_6: ; %endif
-; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2278,18 +2256,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: divergent_cfg:
; GFX1064DAGISEL: ; %bb.0: ; %entry
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1064DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX1064DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2304,45 +2281,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1064DAGISEL-NEXT: ; %bb.5:
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
;
; GFX1064GISEL-LABEL: divergent_cfg:
; GFX1064GISEL: ; %bb.0: ; %entry
+; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX1064GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064GISEL-NEXT: ; %bb.1: ; %else
-; GFX1064GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX1064GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1064GISEL-NEXT: s_mov_b32 s6, s4
; GFX1064GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064GISEL-NEXT: ; %bb.3: ; %if
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: s_brev_b32 s6, 1
; GFX1064GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064GISEL-NEXT: s_max_i32 s6, s6, s8
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064GISEL-NEXT: ; %bb.5:
; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2350,71 +2325,68 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: divergent_cfg:
; GFX1032DAGISEL: ; %bb.0: ; %entry
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc_lo, 15, v0
-; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s3, vcc_lo
+; GFX1032DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1032DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1032DAGISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: s_brev_b32 s1, 1
+; GFX1032DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032DAGISEL-NEXT: s_brev_b32 s2, 1
; GFX1032DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032DAGISEL-NEXT: s_max_i32 s1, s1, s6
-; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032DAGISEL-NEXT: s_max_i32 s2, s2, s6
+; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032DAGISEL-NEXT: ; %bb.5:
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1032DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
; GFX1032GISEL-LABEL: divergent_cfg:
; GFX1032GISEL: ; %bb.0: ; %entry
+; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 16, v0
-; GFX1032GISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1032GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032GISEL-NEXT: ; %bb.1: ; %else
-; GFX1032GISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX1032GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1032GISEL-NEXT: s_mov_b32 s3, s3
; GFX1032GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032GISEL-NEXT: ; %bb.3: ; %if
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: s_brev_b32 s1, 1
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032GISEL-NEXT: s_brev_b32 s3, 1
; GFX1032GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032GISEL-NEXT: s_max_i32 s1, s1, s6
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032GISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032GISEL-NEXT: s_max_i32 s3, s3, s6
+; GFX1032GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032GISEL-NEXT: ; %bb.5:
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2422,20 +2394,20 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: divergent_cfg:
; GFX1164DAGISEL: ; %bb.0: ; %entry
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1164DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1164DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2451,48 +2423,46 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1164DAGISEL-NEXT: ; %bb.5:
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
; GFX1164GISEL-LABEL: divergent_cfg:
; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1164GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1164GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164GISEL-NEXT: ; %bb.1: ; %else
-; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b32 s4, s[4:5], 0x2c
; GFX1164GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1164GISEL-NEXT: s_mov_b32 s6, s4
; GFX1164GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164GISEL-NEXT: ; %bb.3: ; %if
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: s_brev_b32 s6, 1
; GFX1164GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164GISEL-NEXT: s_max_i32 s6, s6, s8
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164GISEL-NEXT: ; %bb.5:
; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2500,77 +2470,75 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: divergent_cfg:
; GFX1132DAGISEL: ; %bb.0: ; %entry
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1132DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1132DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1132DAGISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
; GFX1132DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: s_brev_b32 s1, 1
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: s_brev_b32 s2, 1
; GFX1132DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132DAGISEL-NEXT: s_max_i32 s1, s1, s6
-; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132DAGISEL-NEXT: s_max_i32 s2, s2, s6
+; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132DAGISEL-NEXT: ; %bb.5:
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1132DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
; GFX1132GISEL-LABEL: divergent_cfg:
; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1132GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1132GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132GISEL-NEXT: ; %bb.1: ; %else
-; GFX1132GISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b32 s3, s[4:5], 0x2c
; GFX1132GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1132GISEL-NEXT: s_mov_b32 s3, s3
; GFX1132GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132GISEL-NEXT: ; %bb.3: ; %if
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-NEXT: s_brev_b32 s1, 1
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132GISEL-NEXT: s_brev_b32 s3, 1
; GFX1132GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132GISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132GISEL-NEXT: s_max_i32 s1, s1, s6
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132GISEL-NEXT: s_max_i32 s3, s3, s6
+; GFX1132GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132GISEL-NEXT: ; %bb.5:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
index 2813d9de0f2f1..4c0fd02bfd610 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.min.ll
@@ -15,8 +15,7 @@
define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -37,8 +36,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: uniform_value:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -57,9 +55,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX10DAGISEL-LABEL: uniform_value:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -79,9 +75,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: uniform_value:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -101,9 +95,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: uniform_value:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -370,8 +362,7 @@ entry:
define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_dpp:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -392,8 +383,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX9DAGISEL-LABEL: uniform_value_dpp:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -412,9 +402,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX10DAGISEL-LABEL: uniform_value_dpp:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -434,9 +422,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1164DAGISEL-LABEL: uniform_value_dpp:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -456,9 +442,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1132DAGISEL-LABEL: uniform_value_dpp:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2132,18 +2116,17 @@ entry:
define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: divergent_cfg:
; GFX8DAGISEL: ; %bb.0: ; %entry
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX8DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8DAGISEL-NEXT: ; %bb.3: ; %if
; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2158,46 +2141,44 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-NEXT: ; %bb.5:
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8DAGISEL-NEXT: flat_store_dword v[2:3], v1
; GFX8DAGISEL-NEXT: s_endpgm
;
; GFX8GISEL-LABEL: divergent_cfg:
; GFX8GISEL: ; %bb.0: ; %entry
+; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX8GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX8GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8GISEL-NEXT: ; %bb.1: ; %else
-; GFX8GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX8GISEL-NEXT: ; implicit-def: $vgpr0
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_mov_b32 s2, s2
+; GFX8GISEL-NEXT: s_mov_b32 s6, s4
; GFX8GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8GISEL-NEXT: ; %bb.3: ; %if
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8GISEL-NEXT: s_brev_b32 s6, -2
; GFX8GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8GISEL-NEXT: s_min_i32 s6, s6, s8
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8GISEL-NEXT: ; %bb.5:
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8GISEL-NEXT: .LBB6_6: ; %endif
-; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8GISEL-NEXT: v_mov_b32_e32 v2, s0
@@ -2206,18 +2187,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: divergent_cfg:
; GFX9DAGISEL: ; %bb.0: ; %entry
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX9DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9DAGISEL-NEXT: ; %bb.3: ; %if
; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2232,45 +2212,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX9DAGISEL-NEXT: ; %bb.5:
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
;
; GFX9GISEL-LABEL: divergent_cfg:
; GFX9GISEL: ; %bb.0: ; %entry
+; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX9GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9GISEL-NEXT: ; %bb.1: ; %else
-; GFX9GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX9GISEL-NEXT: ; implicit-def: $vgpr0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_mov_b32 s2, s2
+; GFX9GISEL-NEXT: s_mov_b32 s6, s4
; GFX9GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9GISEL-NEXT: ; %bb.3: ; %if
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9GISEL-NEXT: s_brev_b32 s6, -2
; GFX9GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9GISEL-NEXT: s_min_i32 s6, s6, s8
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9GISEL-NEXT: ; %bb.5:
; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9GISEL-NEXT: .LBB6_6: ; %endif
-; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2278,18 +2256,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: divergent_cfg:
; GFX1064DAGISEL: ; %bb.0: ; %entry
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1064DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX1064DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2304,45 +2281,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1064DAGISEL-NEXT: ; %bb.5:
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
;
; GFX1064GISEL-LABEL: divergent_cfg:
; GFX1064GISEL: ; %bb.0: ; %entry
+; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX1064GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064GISEL-NEXT: ; %bb.1: ; %else
-; GFX1064GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX1064GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1064GISEL-NEXT: s_mov_b32 s6, s4
; GFX1064GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064GISEL-NEXT: ; %bb.3: ; %if
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: s_brev_b32 s6, -2
; GFX1064GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064GISEL-NEXT: s_min_i32 s6, s6, s8
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064GISEL-NEXT: ; %bb.5:
; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2350,71 +2325,68 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: divergent_cfg:
; GFX1032DAGISEL: ; %bb.0: ; %entry
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc_lo, 15, v0
-; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s3, vcc_lo
+; GFX1032DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1032DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1032DAGISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: s_brev_b32 s1, -2
+; GFX1032DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032DAGISEL-NEXT: s_brev_b32 s2, -2
; GFX1032DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032DAGISEL-NEXT: s_min_i32 s1, s1, s6
-; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032DAGISEL-NEXT: s_min_i32 s2, s2, s6
+; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032DAGISEL-NEXT: ; %bb.5:
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1032DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
; GFX1032GISEL-LABEL: divergent_cfg:
; GFX1032GISEL: ; %bb.0: ; %entry
+; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 16, v0
-; GFX1032GISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1032GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032GISEL-NEXT: ; %bb.1: ; %else
-; GFX1032GISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX1032GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1032GISEL-NEXT: s_mov_b32 s3, s3
; GFX1032GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032GISEL-NEXT: ; %bb.3: ; %if
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: s_brev_b32 s1, -2
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032GISEL-NEXT: s_brev_b32 s3, -2
; GFX1032GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032GISEL-NEXT: s_min_i32 s1, s1, s6
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032GISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032GISEL-NEXT: s_min_i32 s3, s3, s6
+; GFX1032GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032GISEL-NEXT: ; %bb.5:
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2422,20 +2394,20 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: divergent_cfg:
; GFX1164DAGISEL: ; %bb.0: ; %entry
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1164DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1164DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2451,48 +2423,46 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1164DAGISEL-NEXT: ; %bb.5:
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
; GFX1164GISEL-LABEL: divergent_cfg:
; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1164GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1164GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164GISEL-NEXT: ; %bb.1: ; %else
-; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b32 s4, s[4:5], 0x2c
; GFX1164GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1164GISEL-NEXT: s_mov_b32 s6, s4
; GFX1164GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164GISEL-NEXT: ; %bb.3: ; %if
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: s_brev_b32 s6, -2
; GFX1164GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164GISEL-NEXT: s_min_i32 s6, s6, s8
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164GISEL-NEXT: ; %bb.5:
; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2500,77 +2470,75 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: divergent_cfg:
; GFX1132DAGISEL: ; %bb.0: ; %entry
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1132DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1132DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1132DAGISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
; GFX1132DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: s_brev_b32 s1, -2
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: s_brev_b32 s2, -2
; GFX1132DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132DAGISEL-NEXT: s_min_i32 s1, s1, s6
-; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132DAGISEL-NEXT: s_min_i32 s2, s2, s6
+; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132DAGISEL-NEXT: ; %bb.5:
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1132DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
; GFX1132GISEL-LABEL: divergent_cfg:
; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1132GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1132GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132GISEL-NEXT: ; %bb.1: ; %else
-; GFX1132GISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b32 s3, s[4:5], 0x2c
; GFX1132GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1132GISEL-NEXT: s_mov_b32 s3, s3
; GFX1132GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132GISEL-NEXT: ; %bb.3: ; %if
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-NEXT: s_brev_b32 s1, -2
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132GISEL-NEXT: s_brev_b32 s3, -2
; GFX1132GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132GISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132GISEL-NEXT: s_min_i32 s1, s1, s6
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132GISEL-NEXT: s_min_i32 s3, s3, s6
+; GFX1132GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132GISEL-NEXT: ; %bb.5:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
index aa9735676d697..2383c7afda409 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.or.ll
@@ -15,8 +15,7 @@
define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -37,8 +36,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: uniform_value:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -57,9 +55,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX10DAGISEL-LABEL: uniform_value:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -79,9 +75,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: uniform_value:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -101,9 +95,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: uniform_value:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -370,8 +362,7 @@ entry:
define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_dpp:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -392,8 +383,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX9DAGISEL-LABEL: uniform_value_dpp:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -412,9 +402,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX10DAGISEL-LABEL: uniform_value_dpp:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -434,9 +422,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1164DAGISEL-LABEL: uniform_value_dpp:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -456,9 +442,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1132DAGISEL-LABEL: uniform_value_dpp:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2000,18 +1984,17 @@ entry:
define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: divergent_cfg:
; GFX8DAGISEL: ; %bb.0: ; %entry
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX8DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8DAGISEL-NEXT: ; %bb.3: ; %if
; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2026,46 +2009,44 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-NEXT: ; %bb.5:
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8DAGISEL-NEXT: flat_store_dword v[2:3], v1
; GFX8DAGISEL-NEXT: s_endpgm
;
; GFX8GISEL-LABEL: divergent_cfg:
; GFX8GISEL: ; %bb.0: ; %entry
+; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX8GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX8GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8GISEL-NEXT: ; %bb.1: ; %else
-; GFX8GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX8GISEL-NEXT: ; implicit-def: $vgpr0
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_mov_b32 s2, s2
+; GFX8GISEL-NEXT: s_mov_b32 s6, s4
; GFX8GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8GISEL-NEXT: ; %bb.3: ; %if
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8GISEL-NEXT: s_mov_b32 s6, 0
; GFX8GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8GISEL-NEXT: s_or_b32 s6, s6, s8
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8GISEL-NEXT: ; %bb.5:
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8GISEL-NEXT: .LBB6_6: ; %endif
-; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8GISEL-NEXT: v_mov_b32_e32 v2, s0
@@ -2074,18 +2055,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: divergent_cfg:
; GFX9DAGISEL: ; %bb.0: ; %entry
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX9DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9DAGISEL-NEXT: ; %bb.3: ; %if
; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2100,45 +2080,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX9DAGISEL-NEXT: ; %bb.5:
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
;
; GFX9GISEL-LABEL: divergent_cfg:
; GFX9GISEL: ; %bb.0: ; %entry
+; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX9GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9GISEL-NEXT: ; %bb.1: ; %else
-; GFX9GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX9GISEL-NEXT: ; implicit-def: $vgpr0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_mov_b32 s2, s2
+; GFX9GISEL-NEXT: s_mov_b32 s6, s4
; GFX9GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9GISEL-NEXT: ; %bb.3: ; %if
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9GISEL-NEXT: s_mov_b32 s6, 0
; GFX9GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9GISEL-NEXT: s_or_b32 s6, s6, s8
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9GISEL-NEXT: ; %bb.5:
; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9GISEL-NEXT: .LBB6_6: ; %endif
-; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2146,18 +2124,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: divergent_cfg:
; GFX1064DAGISEL: ; %bb.0: ; %entry
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1064DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX1064DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2172,45 +2149,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1064DAGISEL-NEXT: ; %bb.5:
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
;
; GFX1064GISEL-LABEL: divergent_cfg:
; GFX1064GISEL: ; %bb.0: ; %entry
+; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX1064GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064GISEL-NEXT: ; %bb.1: ; %else
-; GFX1064GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX1064GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1064GISEL-NEXT: s_mov_b32 s6, s4
; GFX1064GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064GISEL-NEXT: ; %bb.3: ; %if
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: s_mov_b32 s6, 0
; GFX1064GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064GISEL-NEXT: s_or_b32 s6, s6, s8
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064GISEL-NEXT: ; %bb.5:
; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2218,71 +2193,68 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: divergent_cfg:
; GFX1032DAGISEL: ; %bb.0: ; %entry
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc_lo, 15, v0
-; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s3, vcc_lo
+; GFX1032DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1032DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1032DAGISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX1032DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032DAGISEL-NEXT: s_mov_b32 s2, 0
; GFX1032DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032DAGISEL-NEXT: s_or_b32 s1, s1, s6
-; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032DAGISEL-NEXT: s_or_b32 s2, s2, s6
+; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032DAGISEL-NEXT: ; %bb.5:
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1032DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
; GFX1032GISEL-LABEL: divergent_cfg:
; GFX1032GISEL: ; %bb.0: ; %entry
+; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 16, v0
-; GFX1032GISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1032GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032GISEL-NEXT: ; %bb.1: ; %else
-; GFX1032GISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX1032GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1032GISEL-NEXT: s_mov_b32 s3, s3
; GFX1032GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032GISEL-NEXT: ; %bb.3: ; %if
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: s_mov_b32 s1, 0
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032GISEL-NEXT: s_mov_b32 s3, 0
; GFX1032GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032GISEL-NEXT: s_or_b32 s1, s1, s6
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032GISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032GISEL-NEXT: s_or_b32 s3, s3, s6
+; GFX1032GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032GISEL-NEXT: ; %bb.5:
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2290,20 +2262,20 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: divergent_cfg:
; GFX1164DAGISEL: ; %bb.0: ; %entry
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1164DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1164DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2319,48 +2291,46 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1164DAGISEL-NEXT: ; %bb.5:
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
; GFX1164GISEL-LABEL: divergent_cfg:
; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1164GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1164GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164GISEL-NEXT: ; %bb.1: ; %else
-; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b32 s4, s[4:5], 0x2c
; GFX1164GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1164GISEL-NEXT: s_mov_b32 s6, s4
; GFX1164GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164GISEL-NEXT: ; %bb.3: ; %if
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: s_mov_b32 s6, 0
; GFX1164GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164GISEL-NEXT: s_or_b32 s6, s6, s8
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164GISEL-NEXT: ; %bb.5:
; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2368,77 +2338,75 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: divergent_cfg:
; GFX1132DAGISEL: ; %bb.0: ; %entry
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1132DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1132DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1132DAGISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
; GFX1132DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: s_mov_b32 s2, 0
; GFX1132DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132DAGISEL-NEXT: s_or_b32 s1, s1, s6
-; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132DAGISEL-NEXT: s_or_b32 s2, s2, s6
+; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132DAGISEL-NEXT: ; %bb.5:
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1132DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
; GFX1132GISEL-LABEL: divergent_cfg:
; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1132GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1132GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132GISEL-NEXT: ; %bb.1: ; %else
-; GFX1132GISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b32 s3, s[4:5], 0x2c
; GFX1132GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1132GISEL-NEXT: s_mov_b32 s3, s3
; GFX1132GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132GISEL-NEXT: ; %bb.3: ; %if
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s1, 0
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132GISEL-NEXT: s_mov_b32 s3, 0
; GFX1132GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132GISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132GISEL-NEXT: s_or_b32 s1, s1, s6
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132GISEL-NEXT: s_or_b32 s3, s3, s6
+; GFX1132GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132GISEL-NEXT: ; %bb.5:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
index e80a5104b4fe8..1b4309b71051e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.sub.ll
@@ -16,16 +16,15 @@
define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX8DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8DAGISEL-NEXT: s_sub_i32 s0, 0, s2
+; GFX8DAGISEL-NEXT: s_mul_i32 s0, s0, s3
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX8DAGISEL-NEXT: flat_store_dword v[0:1], v2
; GFX8DAGISEL-NEXT: s_endpgm
;
@@ -46,14 +45,13 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: uniform_value:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX9DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_sub_i32 s2, 0, s2
+; GFX9DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
@@ -74,15 +72,13 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: uniform_value:
; GFX1064DAGISEL: ; %bb.0: ; %entry
-; GFX1064DAGISEL-NEXT: s_clause 0x1
-; GFX1064DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1064DAGISEL-NEXT: s_sub_i32 s2, 0, s2
+; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
@@ -104,13 +100,11 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: uniform_value:
; GFX1032DAGISEL: ; %bb.0: ; %entry
-; GFX1032DAGISEL-NEXT: s_clause 0x1
-; GFX1032DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_sub_i32 s2, 0, s2
; GFX1032DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -134,16 +128,14 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: uniform_value:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_sub_i32 s3, 0, s6
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1164DAGISEL-NEXT: s_sub_i32 s2, 0, s2
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
@@ -166,13 +158,11 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: uniform_value:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_sub_i32 s2, 0, s2
; GFX1132DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -481,16 +471,15 @@ entry:
define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_dpp:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX8DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8DAGISEL-NEXT: s_sub_i32 s0, 0, s2
+; GFX8DAGISEL-NEXT: s_mul_i32 s0, s0, s3
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX8DAGISEL-NEXT: flat_store_dword v[0:1], v2
; GFX8DAGISEL-NEXT: s_endpgm
;
@@ -511,14 +500,13 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX9DAGISEL-LABEL: uniform_value_dpp:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX9DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_sub_i32 s2, 0, s2
+; GFX9DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
@@ -539,15 +527,13 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1064DAGISEL-LABEL: uniform_value_dpp:
; GFX1064DAGISEL: ; %bb.0: ; %entry
-; GFX1064DAGISEL-NEXT: s_clause 0x1
-; GFX1064DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1064DAGISEL-NEXT: s_sub_i32 s2, 0, s2
+; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
@@ -569,13 +555,11 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1032DAGISEL-LABEL: uniform_value_dpp:
; GFX1032DAGISEL: ; %bb.0: ; %entry
-; GFX1032DAGISEL-NEXT: s_clause 0x1
-; GFX1032DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_sub_i32 s2, 0, s2
; GFX1032DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -599,16 +583,14 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1164DAGISEL-LABEL: uniform_value_dpp:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_sub_i32 s3, 0, s6
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1164DAGISEL-NEXT: s_sub_i32 s2, 0, s2
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
@@ -631,13 +613,11 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1132DAGISEL-LABEL: uniform_value_dpp:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
-; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_sub_i32 s2, 0, s2
; GFX1132DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
@@ -2398,80 +2378,79 @@ entry:
define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: divergent_cfg:
; GFX8DAGISEL: ; %bb.0: ; %entry
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX8DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX8DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX8DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX8DAGISEL-NEXT: s_sub_i32 s2, 0, s2
+; GFX8DAGISEL-NEXT: s_mul_i32 s6, s2, s3
+; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX8DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX8DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8DAGISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8DAGISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8DAGISEL-NEXT: s_sub_i32 s6, s6, s8
-; GFX8DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8DAGISEL-NEXT: ; %bb.5:
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8DAGISEL-NEXT: flat_store_dword v[2:3], v1
; GFX8DAGISEL-NEXT: s_endpgm
;
; GFX8GISEL-LABEL: divergent_cfg:
; GFX8GISEL: ; %bb.0: ; %entry
+; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX8GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX8GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8GISEL-NEXT: ; %bb.1: ; %else
; GFX8GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX8GISEL-NEXT: ; implicit-def: $vgpr0
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX8GISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX8GISEL-NEXT: s_sub_i32 s5, 0, s6
+; GFX8GISEL-NEXT: s_mul_i32 s6, s5, s4
; GFX8GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8GISEL-NEXT: ; %bb.3: ; %if
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8GISEL-NEXT: s_mov_b32 s6, 0
; GFX8GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8GISEL-NEXT: s_sub_i32 s6, s6, s8
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8GISEL-NEXT: ; %bb.5:
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8GISEL-NEXT: .LBB6_6: ; %endif
-; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8GISEL-NEXT: v_mov_b32_e32 v2, s0
@@ -2480,79 +2459,78 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: divergent_cfg:
; GFX9DAGISEL: ; %bb.0: ; %entry
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX9DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX9DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX9DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX9DAGISEL-NEXT: s_sub_i32 s2, 0, s2
+; GFX9DAGISEL-NEXT: s_mul_i32 s6, s2, s3
+; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX9DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX9DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9DAGISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9DAGISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9DAGISEL-NEXT: s_sub_i32 s6, s6, s8
-; GFX9DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9DAGISEL-NEXT: ; %bb.5:
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
;
; GFX9GISEL-LABEL: divergent_cfg:
; GFX9GISEL: ; %bb.0: ; %entry
+; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX9GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9GISEL-NEXT: ; %bb.1: ; %else
; GFX9GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX9GISEL-NEXT: ; implicit-def: $vgpr0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX9GISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX9GISEL-NEXT: s_sub_i32 s5, 0, s6
+; GFX9GISEL-NEXT: s_mul_i32 s6, s5, s4
; GFX9GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9GISEL-NEXT: ; %bb.3: ; %if
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9GISEL-NEXT: s_mov_b32 s6, 0
; GFX9GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9GISEL-NEXT: s_sub_i32 s6, s6, s8
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9GISEL-NEXT: ; %bb.5:
; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9GISEL-NEXT: .LBB6_6: ; %endif
-; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2560,79 +2538,78 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: divergent_cfg:
; GFX1064DAGISEL: ; %bb.0: ; %entry
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1064DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX1064DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX1064DAGISEL-NEXT: s_sub_i32 s2, 0, s2
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX1064DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX1064DAGISEL-NEXT: s_mul_i32 s6, s2, s3
; GFX1064DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX1064DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064DAGISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064DAGISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064DAGISEL-NEXT: s_sub_i32 s6, s6, s8
-; GFX1064DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064DAGISEL-NEXT: ; %bb.5:
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
;
; GFX1064GISEL-LABEL: divergent_cfg:
; GFX1064GISEL: ; %bb.0: ; %entry
+; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX1064GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064GISEL-NEXT: ; %bb.1: ; %else
; GFX1064GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX1064GISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX1064GISEL-NEXT: s_sub_i32 s5, 0, s6
+; GFX1064GISEL-NEXT: s_mul_i32 s6, s5, s4
; GFX1064GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064GISEL-NEXT: ; %bb.3: ; %if
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: s_mov_b32 s6, 0
; GFX1064GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064GISEL-NEXT: s_sub_i32 s6, s6, s8
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064GISEL-NEXT: ; %bb.5:
; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2640,79 +2617,78 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: divergent_cfg:
; GFX1032DAGISEL: ; %bb.0: ; %entry
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc_lo, 15, v0
-; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s4, vcc_lo
+; GFX1032DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1032DAGISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: s_sub_i32 s1, 0, s1
-; GFX1032DAGISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1032DAGISEL-NEXT: s_sub_i32 s2, 0, s2
+; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX1032DAGISEL-NEXT: s_mul_i32 s3, s2, s3
; GFX1032DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX1032DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032DAGISEL-NEXT: s_mov_b32 s3, 0
; GFX1032DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032DAGISEL-NEXT: s_sub_i32 s1, s1, s6
-; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032DAGISEL-NEXT: s_sub_i32 s3, s3, s6
+; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032DAGISEL-NEXT: ; %bb.5:
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
; GFX1032GISEL-LABEL: divergent_cfg:
; GFX1032GISEL: ; %bb.0: ; %entry
+; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 16, v0
-; GFX1032GISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1032GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032GISEL-NEXT: ; %bb.1: ; %else
-; GFX1032GISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX1032GISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: s_sub_i32 s1, 0, s1
-; GFX1032GISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1032GISEL-NEXT: s_sub_i32 s3, 0, s3
+; GFX1032GISEL-NEXT: s_mul_i32 s3, s3, s4
; GFX1032GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032GISEL-NEXT: ; %bb.3: ; %if
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: s_mov_b32 s1, 0
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032GISEL-NEXT: s_mov_b32 s3, 0
; GFX1032GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032GISEL-NEXT: s_sub_i32 s1, s1, s6
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032GISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032GISEL-NEXT: s_sub_i32 s3, s3, s6
+; GFX1032GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032GISEL-NEXT: ; %bb.5:
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2720,87 +2696,86 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: divergent_cfg:
; GFX1164DAGISEL: ; %bb.0: ; %entry
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1164DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1164DAGISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX1164DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX1164DAGISEL-NEXT: s_sub_i32 s2, 0, s2
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX1164DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_mul_i32 s6, s2, s3
; GFX1164DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX1164DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164DAGISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164DAGISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164DAGISEL-NEXT: s_sub_i32 s6, s6, s8
-; GFX1164DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164DAGISEL-NEXT: ; %bb.5:
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
; GFX1164GISEL-LABEL: divergent_cfg:
; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1164GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1164GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164GISEL-NEXT: ; %bb.1: ; %else
; GFX1164GISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
+; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_sub_i32 s3, 0, s6
-; GFX1164GISEL-NEXT: s_mul_i32 s2, s3, s2
+; GFX1164GISEL-NEXT: s_sub_i32 s5, 0, s6
+; GFX1164GISEL-NEXT: s_mul_i32 s6, s5, s4
; GFX1164GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164GISEL-NEXT: ; %bb.3: ; %if
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: s_mov_b32 s6, 0
; GFX1164GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164GISEL-NEXT: s_sub_i32 s6, s6, s8
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164GISEL-NEXT: ; %bb.5:
; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2808,87 +2783,86 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: divergent_cfg:
; GFX1132DAGISEL: ; %bb.0: ; %entry
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1132DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1132DAGISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: s_sub_i32 s1, 0, s1
-; GFX1132DAGISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1132DAGISEL-NEXT: s_sub_i32 s2, 0, s2
+; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1132DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_mul_i32 s3, s2, s3
; GFX1132DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, 0
; GFX1132DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132DAGISEL-NEXT: s_sub_i32 s1, s1, s6
-; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132DAGISEL-NEXT: s_sub_i32 s3, s3, s6
+; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132DAGISEL-NEXT: ; %bb.5:
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
; GFX1132GISEL-LABEL: divergent_cfg:
; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1132GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1132GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132GISEL-NEXT: ; %bb.1: ; %else
-; GFX1132GISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: s_load_b32 s3, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX1132GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s4, s4
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: s_sub_i32 s1, 0, s1
-; GFX1132GISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1132GISEL-NEXT: s_sub_i32 s3, 0, s3
+; GFX1132GISEL-NEXT: s_mul_i32 s3, s3, s4
; GFX1132GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132GISEL-NEXT: ; %bb.3: ; %if
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s1, 0
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132GISEL-NEXT: s_mov_b32 s3, 0
; GFX1132GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132GISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132GISEL-NEXT: s_sub_i32 s1, s1, s6
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132GISEL-NEXT: s_sub_i32 s3, s3, s6
+; GFX1132GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132GISEL-NEXT: ; %bb.5:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2896,46 +2870,45 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX12DAGISEL-LABEL: divergent_cfg:
; GFX12DAGISEL: ; %bb.0: ; %entry
+; GFX12DAGISEL-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; GFX12DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX12DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX12DAGISEL-NEXT: ; implicit-def: $sgpr1
+; GFX12DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX12DAGISEL-NEXT: ; implicit-def: $sgpr3
; GFX12DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX12DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX12DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX12DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX12DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX12DAGISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
-; GFX12DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX12DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX12DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX12DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
-; GFX12DAGISEL-NEXT: s_sub_co_i32 s1, 0, s1
-; GFX12DAGISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX12DAGISEL-NEXT: s_sub_co_i32 s2, 0, s2
+; GFX12DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX12DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX12DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12DAGISEL-NEXT: s_mul_i32 s3, s2, s3
; GFX12DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX12DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX12DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX12DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
+; GFX12DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX12DAGISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX12DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX12DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX12DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX12DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX12DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX12DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX12DAGISEL-NEXT: s_mov_b32 s3, 0
; GFX12DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX12DAGISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX12DAGISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX12DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX12DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX12DAGISEL-NEXT: s_sub_co_i32 s1, s1, s6
+; GFX12DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX12DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX12DAGISEL-NEXT: s_sub_co_i32 s3, s3, s6
; GFX12DAGISEL-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX12DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX12DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX12DAGISEL-NEXT: ; %bb.5:
-; GFX12DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX12DAGISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX12DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX12DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX12DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX12DAGISEL-NEXT: s_wait_kmcnt 0x0
; GFX12DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX12DAGISEL-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
index 95b99f5475e65..e91771e47e201 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umax.ll
@@ -16,8 +16,7 @@
define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -38,8 +37,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: uniform_value:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -58,9 +56,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX10DAGISEL-LABEL: uniform_value:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -80,9 +76,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: uniform_value:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -102,9 +96,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: uniform_value:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -371,8 +363,7 @@ entry:
define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_dpp:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -393,8 +384,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX9DAGISEL-LABEL: uniform_value_dpp:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -413,9 +403,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX10DAGISEL-LABEL: uniform_value_dpp:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -435,9 +423,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1164DAGISEL-LABEL: uniform_value_dpp:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -457,9 +443,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1132DAGISEL-LABEL: uniform_value_dpp:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2085,18 +2069,17 @@ entry:
define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: divergent_cfg:
; GFX8DAGISEL: ; %bb.0: ; %entry
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX8DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8DAGISEL-NEXT: ; %bb.3: ; %if
; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2111,46 +2094,44 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-NEXT: ; %bb.5:
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8DAGISEL-NEXT: flat_store_dword v[2:3], v1
; GFX8DAGISEL-NEXT: s_endpgm
;
; GFX8GISEL-LABEL: divergent_cfg:
; GFX8GISEL: ; %bb.0: ; %entry
+; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX8GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX8GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8GISEL-NEXT: ; %bb.1: ; %else
-; GFX8GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX8GISEL-NEXT: ; implicit-def: $vgpr0
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_mov_b32 s2, s2
+; GFX8GISEL-NEXT: s_mov_b32 s6, s4
; GFX8GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8GISEL-NEXT: ; %bb.3: ; %if
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8GISEL-NEXT: s_mov_b32 s6, 0
; GFX8GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8GISEL-NEXT: s_max_u32 s6, s6, s8
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8GISEL-NEXT: ; %bb.5:
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8GISEL-NEXT: .LBB6_6: ; %endif
-; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8GISEL-NEXT: v_mov_b32_e32 v2, s0
@@ -2159,18 +2140,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: divergent_cfg:
; GFX9DAGISEL: ; %bb.0: ; %entry
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX9DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9DAGISEL-NEXT: ; %bb.3: ; %if
; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2185,45 +2165,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX9DAGISEL-NEXT: ; %bb.5:
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
;
; GFX9GISEL-LABEL: divergent_cfg:
; GFX9GISEL: ; %bb.0: ; %entry
+; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX9GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9GISEL-NEXT: ; %bb.1: ; %else
-; GFX9GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX9GISEL-NEXT: ; implicit-def: $vgpr0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_mov_b32 s2, s2
+; GFX9GISEL-NEXT: s_mov_b32 s6, s4
; GFX9GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9GISEL-NEXT: ; %bb.3: ; %if
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9GISEL-NEXT: s_mov_b32 s6, 0
; GFX9GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9GISEL-NEXT: s_max_u32 s6, s6, s8
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9GISEL-NEXT: ; %bb.5:
; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9GISEL-NEXT: .LBB6_6: ; %endif
-; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2231,18 +2209,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: divergent_cfg:
; GFX1064DAGISEL: ; %bb.0: ; %entry
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1064DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX1064DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2257,45 +2234,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1064DAGISEL-NEXT: ; %bb.5:
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
;
; GFX1064GISEL-LABEL: divergent_cfg:
; GFX1064GISEL: ; %bb.0: ; %entry
+; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX1064GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064GISEL-NEXT: ; %bb.1: ; %else
-; GFX1064GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX1064GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1064GISEL-NEXT: s_mov_b32 s6, s4
; GFX1064GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064GISEL-NEXT: ; %bb.3: ; %if
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: s_mov_b32 s6, 0
; GFX1064GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064GISEL-NEXT: s_max_u32 s6, s6, s8
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064GISEL-NEXT: ; %bb.5:
; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2303,71 +2278,68 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: divergent_cfg:
; GFX1032DAGISEL: ; %bb.0: ; %entry
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc_lo, 15, v0
-; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s3, vcc_lo
+; GFX1032DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1032DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1032DAGISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX1032DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032DAGISEL-NEXT: s_mov_b32 s2, 0
; GFX1032DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032DAGISEL-NEXT: s_max_u32 s1, s1, s6
-; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032DAGISEL-NEXT: s_max_u32 s2, s2, s6
+; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032DAGISEL-NEXT: ; %bb.5:
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1032DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
; GFX1032GISEL-LABEL: divergent_cfg:
; GFX1032GISEL: ; %bb.0: ; %entry
+; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 16, v0
-; GFX1032GISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1032GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032GISEL-NEXT: ; %bb.1: ; %else
-; GFX1032GISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX1032GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1032GISEL-NEXT: s_mov_b32 s3, s3
; GFX1032GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032GISEL-NEXT: ; %bb.3: ; %if
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: s_mov_b32 s1, 0
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032GISEL-NEXT: s_mov_b32 s3, 0
; GFX1032GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032GISEL-NEXT: s_max_u32 s1, s1, s6
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032GISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032GISEL-NEXT: s_max_u32 s3, s3, s6
+; GFX1032GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032GISEL-NEXT: ; %bb.5:
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2375,20 +2347,20 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: divergent_cfg:
; GFX1164DAGISEL: ; %bb.0: ; %entry
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1164DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1164DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2404,48 +2376,46 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1164DAGISEL-NEXT: ; %bb.5:
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
; GFX1164GISEL-LABEL: divergent_cfg:
; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1164GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1164GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164GISEL-NEXT: ; %bb.1: ; %else
-; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b32 s4, s[4:5], 0x2c
; GFX1164GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1164GISEL-NEXT: s_mov_b32 s6, s4
; GFX1164GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164GISEL-NEXT: ; %bb.3: ; %if
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: s_mov_b32 s6, 0
; GFX1164GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164GISEL-NEXT: s_max_u32 s6, s6, s8
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164GISEL-NEXT: ; %bb.5:
; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2453,77 +2423,75 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: divergent_cfg:
; GFX1132DAGISEL: ; %bb.0: ; %entry
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1132DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1132DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1132DAGISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
; GFX1132DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: s_mov_b32 s2, 0
; GFX1132DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132DAGISEL-NEXT: s_max_u32 s1, s1, s6
-; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132DAGISEL-NEXT: s_max_u32 s2, s2, s6
+; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132DAGISEL-NEXT: ; %bb.5:
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1132DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
; GFX1132GISEL-LABEL: divergent_cfg:
; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1132GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1132GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132GISEL-NEXT: ; %bb.1: ; %else
-; GFX1132GISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b32 s3, s[4:5], 0x2c
; GFX1132GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1132GISEL-NEXT: s_mov_b32 s3, s3
; GFX1132GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132GISEL-NEXT: ; %bb.3: ; %if
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s1, 0
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132GISEL-NEXT: s_mov_b32 s3, 0
; GFX1132GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132GISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132GISEL-NEXT: s_max_u32 s1, s1, s6
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132GISEL-NEXT: s_max_u32 s3, s3, s6
+; GFX1132GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132GISEL-NEXT: ; %bb.5:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
index 36c860ea7f4d7..d31e571d77ac0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.umin.ll
@@ -16,8 +16,7 @@
define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -38,8 +37,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: uniform_value:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -58,9 +56,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX10DAGISEL-LABEL: uniform_value:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -80,9 +76,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: uniform_value:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -102,9 +96,7 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: uniform_value:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -371,8 +363,7 @@ entry:
define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_dpp:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
@@ -393,8 +384,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX9DAGISEL-LABEL: uniform_value_dpp:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -413,9 +403,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX10DAGISEL-LABEL: uniform_value_dpp:
; GFX10DAGISEL: ; %bb.0: ; %entry
-; GFX10DAGISEL-NEXT: s_clause 0x1
-; GFX10DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX10DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX10DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -435,9 +423,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1164DAGISEL-LABEL: uniform_value_dpp:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
@@ -457,9 +443,7 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1132DAGISEL-LABEL: uniform_value_dpp:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2085,18 +2069,17 @@ entry:
define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: divergent_cfg:
; GFX8DAGISEL: ; %bb.0: ; %entry
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX8DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX8DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8DAGISEL-NEXT: ; %bb.3: ; %if
; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2111,46 +2094,44 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-NEXT: ; %bb.5:
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8DAGISEL-NEXT: flat_store_dword v[2:3], v1
; GFX8DAGISEL-NEXT: s_endpgm
;
; GFX8GISEL-LABEL: divergent_cfg:
; GFX8GISEL: ; %bb.0: ; %entry
+; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX8GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX8GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8GISEL-NEXT: ; %bb.1: ; %else
-; GFX8GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX8GISEL-NEXT: ; implicit-def: $vgpr0
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_mov_b32 s2, s2
+; GFX8GISEL-NEXT: s_mov_b32 s6, s4
; GFX8GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8GISEL-NEXT: ; %bb.3: ; %if
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8GISEL-NEXT: s_mov_b32 s6, -1
; GFX8GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8GISEL-NEXT: s_min_u32 s6, s6, s8
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8GISEL-NEXT: ; %bb.5:
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8GISEL-NEXT: .LBB6_6: ; %endif
-; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8GISEL-NEXT: v_mov_b32_e32 v2, s0
@@ -2159,18 +2140,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: divergent_cfg:
; GFX9DAGISEL: ; %bb.0: ; %entry
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX9DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX9DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9DAGISEL-NEXT: ; %bb.3: ; %if
; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2185,45 +2165,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX9DAGISEL-NEXT: ; %bb.5:
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
;
; GFX9GISEL-LABEL: divergent_cfg:
; GFX9GISEL: ; %bb.0: ; %entry
+; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX9GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9GISEL-NEXT: ; %bb.1: ; %else
-; GFX9GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX9GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX9GISEL-NEXT: ; implicit-def: $vgpr0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_mov_b32 s2, s2
+; GFX9GISEL-NEXT: s_mov_b32 s6, s4
; GFX9GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9GISEL-NEXT: ; %bb.3: ; %if
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9GISEL-NEXT: s_mov_b32 s6, -1
; GFX9GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9GISEL-NEXT: s_min_u32 s6, s6, s8
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9GISEL-NEXT: ; %bb.5:
; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9GISEL-NEXT: .LBB6_6: ; %endif
-; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2231,18 +2209,17 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: divergent_cfg:
; GFX1064DAGISEL: ; %bb.0: ; %entry
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1064DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX1064DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2257,45 +2234,43 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1064DAGISEL-NEXT: ; %bb.5:
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
;
; GFX1064GISEL-LABEL: divergent_cfg:
; GFX1064GISEL: ; %bb.0: ; %entry
+; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX1064GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064GISEL-NEXT: ; %bb.1: ; %else
-; GFX1064GISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX1064GISEL-NEXT: s_load_dword s4, s[4:5], 0x2c
; GFX1064GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1064GISEL-NEXT: s_mov_b32 s6, s4
; GFX1064GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064GISEL-NEXT: ; %bb.3: ; %if
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: s_mov_b32 s6, -1
; GFX1064GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064GISEL-NEXT: s_min_u32 s6, s6, s8
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064GISEL-NEXT: ; %bb.5:
; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2303,71 +2278,68 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: divergent_cfg:
; GFX1032DAGISEL: ; %bb.0: ; %entry
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc_lo, 15, v0
-; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s3, vcc_lo
+; GFX1032DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1032DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1032DAGISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: s_mov_b32 s1, -1
+; GFX1032DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032DAGISEL-NEXT: s_mov_b32 s2, -1
; GFX1032DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032DAGISEL-NEXT: s_min_u32 s1, s1, s6
-; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032DAGISEL-NEXT: s_min_u32 s2, s2, s6
+; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032DAGISEL-NEXT: ; %bb.5:
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1032DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
; GFX1032GISEL-LABEL: divergent_cfg:
; GFX1032GISEL: ; %bb.0: ; %entry
+; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 16, v0
-; GFX1032GISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1032GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032GISEL-NEXT: ; %bb.1: ; %else
-; GFX1032GISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
; GFX1032GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1032GISEL-NEXT: s_mov_b32 s3, s3
; GFX1032GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032GISEL-NEXT: ; %bb.3: ; %if
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: s_mov_b32 s1, -1
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032GISEL-NEXT: s_mov_b32 s3, -1
; GFX1032GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032GISEL-NEXT: s_min_u32 s1, s1, s6
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032GISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032GISEL-NEXT: s_min_u32 s3, s3, s6
+; GFX1032GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032GISEL-NEXT: ; %bb.5:
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2375,20 +2347,20 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: divergent_cfg:
; GFX1164DAGISEL: ; %bb.0: ; %entry
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1164DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1164DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
; GFX1164DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[4:5], s[4:5]
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164DAGISEL-NEXT: ; %bb.3: ; %if
; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
@@ -2404,48 +2376,46 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1164DAGISEL-NEXT: ; %bb.5:
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
; GFX1164GISEL-LABEL: divergent_cfg:
; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1164GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1164GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164GISEL-NEXT: ; %bb.1: ; %else
-; GFX1164GISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1164GISEL-NEXT: s_load_b32 s4, s[4:5], 0x2c
; GFX1164GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_mov_b32 s2, s2
+; GFX1164GISEL-NEXT: s_mov_b32 s6, s4
; GFX1164GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164GISEL-NEXT: ; %bb.3: ; %if
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: s_mov_b32 s6, -1
; GFX1164GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164GISEL-NEXT: s_min_u32 s6, s6, s8
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164GISEL-NEXT: ; %bb.5:
; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2453,77 +2423,75 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: divergent_cfg:
; GFX1132DAGISEL: ; %bb.0: ; %entry
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1132DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_xor_b32 s3, exec_lo, s3
; GFX1132DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1132DAGISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
; GFX1132DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132DAGISEL-NEXT: ; %bb.2: ; %Flow
-; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s3, s3
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
+; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: s_mov_b32 s1, -1
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: s_mov_b32 s2, -1
; GFX1132DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132DAGISEL-NEXT: s_min_u32 s1, s1, s6
-; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132DAGISEL-NEXT: s_min_u32 s2, s2, s6
+; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132DAGISEL-NEXT: ; %bb.5:
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1132DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
; GFX1132GISEL-LABEL: divergent_cfg:
; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1132GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1132GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132GISEL-NEXT: ; %bb.1: ; %else
-; GFX1132GISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_load_b32 s3, s[4:5], 0x2c
; GFX1132GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: s_mov_b32 s1, s1
+; GFX1132GISEL-NEXT: s_mov_b32 s3, s3
; GFX1132GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132GISEL-NEXT: ; %bb.3: ; %if
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s1, -1
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132GISEL-NEXT: s_mov_b32 s3, -1
; GFX1132GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132GISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132GISEL-NEXT: s_min_u32 s1, s1, s6
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132GISEL-NEXT: s_min_u32 s3, s3, s6
+; GFX1132GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132GISEL-NEXT: ; %bb.5:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
index 057abf6a2d48b..7af8483b1a48f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.reduce.xor.ll
@@ -15,16 +15,15 @@
define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX8DAGISEL-NEXT: s_and_b32 s2, s2, 1
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX8DAGISEL-NEXT: s_and_b32 s3, s3, 1
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8DAGISEL-NEXT: s_mul_i32 s0, s2, s3
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX8DAGISEL-NEXT: flat_store_dword v[0:1], v2
; GFX8DAGISEL-NEXT: s_endpgm
;
@@ -45,14 +44,13 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: uniform_value:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9DAGISEL-NEXT: s_and_b32 s2, s2, 1
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX9DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
@@ -73,15 +71,13 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: uniform_value:
; GFX1064DAGISEL: ; %bb.0: ; %entry
-; GFX1064DAGISEL-NEXT: s_clause 0x1
-; GFX1064DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064DAGISEL-NEXT: s_and_b32 s2, s2, 1
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1064DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
@@ -103,14 +99,12 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: uniform_value:
; GFX1032DAGISEL: ; %bb.0: ; %entry
-; GFX1032DAGISEL-NEXT: s_clause 0x1
-; GFX1032DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1032DAGISEL-NEXT: s_and_b32 s3, s3, 1
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -133,16 +127,15 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: uniform_value:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164DAGISEL-NEXT: s_and_b32 s2, s2, 1
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s2, s3
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
@@ -165,16 +158,14 @@ define amdgpu_kernel void @uniform_value(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: uniform_value:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1132DAGISEL-NEXT: s_and_b32 s3, s3, 1
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: s_mul_i32 s2, s2, s3
-; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
@@ -446,16 +437,15 @@ entry:
define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: uniform_value_dpp:
; GFX8DAGISEL: ; %bb.0: ; %entry
-; GFX8DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX8DAGISEL-NEXT: s_and_b32 s2, s2, 1
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX8DAGISEL-NEXT: s_and_b32 s3, s3, 1
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v0, s0
+; GFX8DAGISEL-NEXT: s_mul_i32 s0, s2, s3
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s2
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX8DAGISEL-NEXT: flat_store_dword v[0:1], v2
; GFX8DAGISEL-NEXT: s_endpgm
;
@@ -476,14 +466,13 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX9DAGISEL-LABEL: uniform_value_dpp:
; GFX9DAGISEL: ; %bb.0: ; %entry
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9DAGISEL-NEXT: s_and_b32 s2, s2, 1
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX9DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
+; GFX9DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
@@ -504,15 +493,13 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1064DAGISEL-LABEL: uniform_value_dpp:
; GFX1064DAGISEL: ; %bb.0: ; %entry
-; GFX1064DAGISEL-NEXT: s_clause 0x1
-; GFX1064DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064DAGISEL-NEXT: s_and_b32 s2, s2, 1
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1064DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
@@ -534,14 +521,12 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1032DAGISEL-LABEL: uniform_value_dpp:
; GFX1032DAGISEL: ; %bb.0: ; %entry
-; GFX1032DAGISEL-NEXT: s_clause 0x1
-; GFX1032DAGISEL-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1032DAGISEL-NEXT: s_and_b32 s3, s3, 1
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: s_mul_i32 s2, s2, s3
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -564,16 +549,15 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1164DAGISEL-LABEL: uniform_value_dpp:
; GFX1164DAGISEL: ; %bb.0: ; %entry
-; GFX1164DAGISEL-NEXT: s_clause 0x1
-; GFX1164DAGISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1164DAGISEL-NEXT: s_and_b32 s2, s2, 1
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[4:5]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s2, s3
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
@@ -596,16 +580,14 @@ define amdgpu_kernel void @uniform_value_dpp(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1132DAGISEL-LABEL: uniform_value_dpp:
; GFX1132DAGISEL: ; %bb.0: ; %entry
-; GFX1132DAGISEL-NEXT: s_clause 0x1
-; GFX1132DAGISEL-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
; GFX1132DAGISEL-NEXT: s_and_b32 s3, s3, 1
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: s_mul_i32 s2, s2, s3
-; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1132DAGISEL-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
@@ -2152,80 +2134,79 @@ entry:
define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
; GFX8DAGISEL-LABEL: divergent_cfg:
; GFX8DAGISEL: ; %bb.0: ; %entry
+; GFX8DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8DAGISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX8DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX8DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX8DAGISEL-NEXT: s_and_b32 s2, s2, 1
-; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX8DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX8DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX8DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX8DAGISEL-NEXT: s_mul_i32 s6, s2, s3
+; GFX8DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX8DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX8DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX8DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8DAGISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8DAGISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8DAGISEL-NEXT: s_xor_b32 s6, s6, s8
-; GFX8DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8DAGISEL-NEXT: ; %bb.5:
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX8DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8DAGISEL-NEXT: v_mov_b32_e32 v2, s0
+; GFX8DAGISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8DAGISEL-NEXT: flat_store_dword v[2:3], v1
; GFX8DAGISEL-NEXT: s_endpgm
;
; GFX8GISEL-LABEL: divergent_cfg:
; GFX8GISEL: ; %bb.0: ; %entry
+; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX8GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX8GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX8GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX8GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX8GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX8GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX8GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX8GISEL-NEXT: ; %bb.1: ; %else
; GFX8GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX8GISEL-NEXT: s_and_b32 s2, s2, 1
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GFX8GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX8GISEL-NEXT: s_and_b32 s4, s4, 1
; GFX8GISEL-NEXT: ; implicit-def: $vgpr0
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8GISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX8GISEL-NEXT: s_mul_i32 s6, s6, s4
; GFX8GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX8GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX8GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX8GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX8GISEL-NEXT: ; %bb.3: ; %if
-; GFX8GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX8GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX8GISEL-NEXT: s_mov_b32 s6, 0
; GFX8GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX8GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX8GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX8GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX8GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX8GISEL-NEXT: s_xor_b32 s6, s6, s8
-; GFX8GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX8GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX8GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX8GISEL-NEXT: ; %bb.5:
; GFX8GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX8GISEL-NEXT: .LBB6_6: ; %endif
-; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX8GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX8GISEL-NEXT: v_mov_b32_e32 v2, s0
@@ -2234,79 +2215,78 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX9DAGISEL-LABEL: divergent_cfg:
; GFX9DAGISEL: ; %bb.0: ; %entry
+; GFX9DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9DAGISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX9DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX9DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9DAGISEL-NEXT: s_and_b32 s2, s2, 1
-; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX9DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX9DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX9DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX9DAGISEL-NEXT: s_mul_i32 s6, s2, s3
+; GFX9DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX9DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX9DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX9DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9DAGISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9DAGISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9DAGISEL-NEXT: s_xor_b32 s6, s6, s8
-; GFX9DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9DAGISEL-NEXT: ; %bb.5:
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX9DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9DAGISEL-NEXT: s_endpgm
;
; GFX9GISEL-LABEL: divergent_cfg:
; GFX9GISEL: ; %bb.0: ; %entry
+; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX9GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX9GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX9GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX9GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX9GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX9GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX9GISEL-NEXT: ; %bb.1: ; %else
; GFX9GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
-; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX9GISEL-NEXT: s_and_b32 s2, s2, 1
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
+; GFX9GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX9GISEL-NEXT: s_and_b32 s4, s4, 1
; GFX9GISEL-NEXT: ; implicit-def: $vgpr0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9GISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX9GISEL-NEXT: s_mul_i32 s6, s6, s4
; GFX9GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX9GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX9GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX9GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX9GISEL-NEXT: ; %bb.3: ; %if
-; GFX9GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX9GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX9GISEL-NEXT: s_mov_b32 s6, 0
; GFX9GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX9GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX9GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX9GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX9GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX9GISEL-NEXT: s_xor_b32 s6, s6, s8
-; GFX9GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX9GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX9GISEL-NEXT: ; %bb.5:
; GFX9GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX9GISEL-NEXT: .LBB6_6: ; %endif
-; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX9GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX9GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX9GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2314,79 +2294,78 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1064DAGISEL-LABEL: divergent_cfg:
; GFX1064DAGISEL: ; %bb.0: ; %entry
+; GFX1064DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc, 15, v0
-; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064DAGISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064DAGISEL-NEXT: s_and_saveexec_b64 s[4:5], vcc
+; GFX1064DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1064DAGISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1064DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064DAGISEL-NEXT: s_and_b32 s2, s2, 1
; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1064DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX1064DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX1064DAGISEL-NEXT: s_mul_i32 s6, s2, s3
; GFX1064DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1064DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1064DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX1064DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064DAGISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064DAGISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064DAGISEL-NEXT: s_xor_b32 s6, s6, s8
-; GFX1064DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064DAGISEL-NEXT: ; %bb.5:
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1064DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1064DAGISEL-NEXT: s_endpgm
;
; GFX1064GISEL-LABEL: divergent_cfg:
; GFX1064GISEL: ; %bb.0: ; %entry
+; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1064GISEL-NEXT: v_cmp_le_u32_e32 vcc, 16, v0
-; GFX1064GISEL-NEXT: ; implicit-def: $sgpr2
-; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX1064GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1064GISEL-NEXT: ; implicit-def: $sgpr6
+; GFX1064GISEL-NEXT: s_and_saveexec_b64 s[2:3], vcc
+; GFX1064GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1064GISEL-NEXT: ; %bb.1: ; %else
; GFX1064GISEL-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1064GISEL-NEXT: s_and_b32 s2, s2, 1
+; GFX1064GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1064GISEL-NEXT: s_and_b32 s4, s4, 1
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064GISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1064GISEL-NEXT: s_mul_i32 s6, s6, s4
; GFX1064GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1064GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1064GISEL-NEXT: ; %bb.3: ; %if
-; GFX1064GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1064GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1064GISEL-NEXT: s_mov_b32 s6, 0
; GFX1064GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[2:3]
+; GFX1064GISEL-NEXT: s_ff1_i32_b64 s7, s[4:5]
; GFX1064GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1064GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1064GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1064GISEL-NEXT: s_xor_b32 s6, s6, s8
-; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1064GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1064GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1064GISEL-NEXT: ; %bb.5:
; GFX1064GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1064GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1064GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1064GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1064GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2394,79 +2373,78 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1032DAGISEL-LABEL: divergent_cfg:
; GFX1032DAGISEL: ; %bb.0: ; %entry
+; GFX1032DAGISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032DAGISEL-NEXT: v_cmp_lt_u32_e32 vcc_lo, 15, v0
-; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032DAGISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032DAGISEL-NEXT: s_and_saveexec_b32 s4, vcc_lo
+; GFX1032DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1032DAGISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1032DAGISEL-NEXT: s_and_b32 s2, s2, 1
; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032DAGISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1032DAGISEL-NEXT: s_mov_b32 s3, exec_lo
+; GFX1032DAGISEL-NEXT: ; implicit-def: $vgpr0
+; GFX1032DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1032DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX1032DAGISEL-NEXT: s_mul_i32 s3, s2, s3
; GFX1032DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1032DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX1032DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032DAGISEL-NEXT: s_mov_b32 s3, 0
; GFX1032DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032DAGISEL-NEXT: s_xor_b32 s1, s1, s6
-; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032DAGISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032DAGISEL-NEXT: s_xor_b32 s3, s3, s6
+; GFX1032DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032DAGISEL-NEXT: ; %bb.5:
-; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032DAGISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1032DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032DAGISEL-NEXT: global_store_dword v0, v1, s[0:1]
; GFX1032DAGISEL-NEXT: s_endpgm
;
; GFX1032GISEL-LABEL: divergent_cfg:
; GFX1032GISEL: ; %bb.0: ; %entry
+; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX1032GISEL-NEXT: v_cmp_le_u32_e32 vcc_lo, 16, v0
-; GFX1032GISEL-NEXT: ; implicit-def: $sgpr1
-; GFX1032GISEL-NEXT: s_and_saveexec_b32 s0, vcc_lo
-; GFX1032GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1032GISEL-NEXT: ; implicit-def: $sgpr3
+; GFX1032GISEL-NEXT: s_and_saveexec_b32 s2, vcc_lo
+; GFX1032GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1032GISEL-NEXT: ; %bb.1: ; %else
-; GFX1032GISEL-NEXT: s_load_dword s1, s[4:5], 0x2c
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1032GISEL-NEXT: s_load_dword s3, s[4:5], 0x2c
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX1032GISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1032GISEL-NEXT: s_and_b32 s2, s2, 1
+; GFX1032GISEL-NEXT: s_bcnt1_i32_b32 s4, s4
+; GFX1032GISEL-NEXT: s_and_b32 s4, s4, 1
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032GISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1032GISEL-NEXT: s_mul_i32 s3, s3, s4
; GFX1032GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1032GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1032GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1032GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1032GISEL-NEXT: ; %bb.3: ; %if
-; GFX1032GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1032GISEL-NEXT: s_mov_b32 s1, 0
+; GFX1032GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1032GISEL-NEXT: s_mov_b32 s3, 0
; GFX1032GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1032GISEL-NEXT: s_ff1_i32_b32 s3, s2
-; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1032GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1032GISEL-NEXT: s_xor_b32 s1, s1, s6
-; GFX1032GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1032GISEL-NEXT: s_ff1_i32_b32 s5, s4
+; GFX1032GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1032GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1032GISEL-NEXT: s_xor_b32 s3, s3, s6
+; GFX1032GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1032GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1032GISEL-NEXT: ; %bb.5:
-; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1032GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1032GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1032GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1032GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1032GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032GISEL-NEXT: global_store_dword v0, v1, s[0:1]
@@ -2474,87 +2452,86 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1164DAGISEL-LABEL: divergent_cfg:
; GFX1164DAGISEL: ; %bb.0: ; %entry
+; GFX1164DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1164DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
+; GFX1164DAGISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1164DAGISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1164DAGISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[6:7], exec
; GFX1164DAGISEL-NEXT: ; implicit-def: $vgpr0
-; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164DAGISEL-NEXT: s_and_b32 s2, s2, 1
; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164DAGISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1164DAGISEL-NEXT: s_bcnt1_i32_b64 s3, s[6:7]
+; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1164DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX1164DAGISEL-NEXT: s_mul_i32 s6, s2, s3
; GFX1164DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1164DAGISEL-NEXT: s_or_saveexec_b64 s[2:3], s[4:5]
+; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164DAGISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1164DAGISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164DAGISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164DAGISEL-NEXT: s_mov_b32 s6, 0
; GFX1164DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164DAGISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164DAGISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164DAGISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164DAGISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164DAGISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164DAGISEL-NEXT: s_xor_b32 s6, s6, s8
-; GFX1164DAGISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164DAGISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164DAGISEL-NEXT: ; %bb.5:
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164DAGISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1164DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1164DAGISEL-NEXT: s_endpgm
;
; GFX1164GISEL-LABEL: divergent_cfg:
; GFX1164GISEL: ; %bb.0: ; %entry
+; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1164GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1164GISEL-NEXT: s_mov_b64 s[0:1], exec
-; GFX1164GISEL-NEXT: ; implicit-def: $sgpr2
+; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: ; implicit-def: $sgpr6
; GFX1164GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1164GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1164GISEL-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
+; GFX1164GISEL-NEXT: s_xor_b64 s[2:3], exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1164GISEL-NEXT: ; %bb.1: ; %else
; GFX1164GISEL-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s2, s[2:3]
-; GFX1164GISEL-NEXT: s_and_b32 s2, s2, 1
+; GFX1164GISEL-NEXT: s_bcnt1_i32_b64 s4, s[4:5]
+; GFX1164GISEL-NEXT: s_and_b32 s4, s4, 1
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1164GISEL-NEXT: s_mul_i32 s2, s6, s2
+; GFX1164GISEL-NEXT: s_mul_i32 s6, s6, s4
; GFX1164GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[0:1], s[0:1]
-; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s2
-; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[0:1]
+; GFX1164GISEL-NEXT: s_or_saveexec_b64 s[2:3], s[2:3]
+; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
+; GFX1164GISEL-NEXT: s_xor_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1164GISEL-NEXT: ; %bb.3: ; %if
-; GFX1164GISEL-NEXT: s_mov_b64 s[2:3], exec
+; GFX1164GISEL-NEXT: s_mov_b64 s[4:5], exec
; GFX1164GISEL-NEXT: s_mov_b32 s6, 0
; GFX1164GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[2:3]
+; GFX1164GISEL-NEXT: s_ctz_i32_b64 s7, s[4:5]
; GFX1164GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1164GISEL-NEXT: v_readlane_b32 s8, v0, s7
-; GFX1164GISEL-NEXT: s_bitset0_b64 s[2:3], s7
+; GFX1164GISEL-NEXT: s_bitset0_b64 s[4:5], s7
; GFX1164GISEL-NEXT: s_xor_b32 s6, s6, s8
-; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[2:3], 0
+; GFX1164GISEL-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX1164GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1164GISEL-NEXT: ; %bb.5:
; GFX1164GISEL-NEXT: v_mov_b32_e32 v1, s6
; GFX1164GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX1164GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1164GISEL-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX1164GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1164GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1164GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -2562,87 +2539,87 @@ define amdgpu_kernel void @divergent_cfg(ptr addrspace(1) %out, i32 %in) #0 {
;
; GFX1132DAGISEL-LABEL: divergent_cfg:
; GFX1132DAGISEL: ; %bb.0: ; %entry
+; GFX1132DAGISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1132DAGISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132DAGISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132DAGISEL-NEXT: v_cmpx_lt_u32_e32 15, v0
-; GFX1132DAGISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_xor_b32 s4, exec_lo, s4
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132DAGISEL-NEXT: ; %bb.1: ; %else
-; GFX1132DAGISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, exec_lo
; GFX1132DAGISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1132DAGISEL-NEXT: s_and_b32 s2, s2, 1
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132DAGISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1132DAGISEL-NEXT: s_bcnt1_i32_b32 s3, s3
+; GFX1132DAGISEL-NEXT: s_and_b32 s3, s3, 1
+; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1132DAGISEL-NEXT: s_mul_i32 s3, s2, s3
; GFX1132DAGISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1132DAGISEL-NEXT: s_or_saveexec_b32 s2, s4
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132DAGISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132DAGISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132DAGISEL-NEXT: ; %bb.3: ; %if
-; GFX1132DAGISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132DAGISEL-NEXT: s_mov_b32 s1, 0
+; GFX1132DAGISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132DAGISEL-NEXT: s_mov_b32 s3, 0
; GFX1132DAGISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132DAGISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132DAGISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132DAGISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132DAGISEL-NEXT: s_xor_b32 s1, s1, s6
-; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132DAGISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132DAGISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132DAGISEL-NEXT: s_xor_b32 s3, s3, s6
+; GFX1132DAGISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132DAGISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132DAGISEL-NEXT: ; %bb.5:
-; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132DAGISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132DAGISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132DAGISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132DAGISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1132DAGISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132DAGISEL-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX1132DAGISEL-NEXT: s_endpgm
;
; GFX1132GISEL-LABEL: divergent_cfg:
; GFX1132GISEL: ; %bb.0: ; %entry
+; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1132GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1132GISEL-NEXT: s_mov_b32 s0, exec_lo
-; GFX1132GISEL-NEXT: ; implicit-def: $sgpr1
+; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: ; implicit-def: $sgpr3
; GFX1132GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1132GISEL-NEXT: v_cmpx_le_u32_e32 16, v0
-; GFX1132GISEL-NEXT: s_xor_b32 s0, exec_lo, s0
+; GFX1132GISEL-NEXT: s_xor_b32 s2, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_2
; GFX1132GISEL-NEXT: ; %bb.1: ; %else
-; GFX1132GISEL-NEXT: s_load_b32 s1, s[4:5], 0x2c
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
+; GFX1132GISEL-NEXT: s_load_b32 s3, s[4:5], 0x2c
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
; GFX1132GISEL-NEXT: ; implicit-def: $vgpr0
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s2, s2
-; GFX1132GISEL-NEXT: s_and_b32 s2, s2, 1
+; GFX1132GISEL-NEXT: s_bcnt1_i32_b32 s4, s4
+; GFX1132GISEL-NEXT: s_and_b32 s4, s4, 1
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1132GISEL-NEXT: s_mul_i32 s1, s1, s2
+; GFX1132GISEL-NEXT: s_mul_i32 s3, s3, s4
; GFX1132GISEL-NEXT: .LBB6_2: ; %Flow
-; GFX1132GISEL-NEXT: s_or_saveexec_b32 s0, s0
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s0
+; GFX1132GISEL-NEXT: s_or_saveexec_b32 s2, s2
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
+; GFX1132GISEL-NEXT: s_xor_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: s_cbranch_execz .LBB6_6
; GFX1132GISEL-NEXT: ; %bb.3: ; %if
-; GFX1132GISEL-NEXT: s_mov_b32 s2, exec_lo
-; GFX1132GISEL-NEXT: s_mov_b32 s1, 0
+; GFX1132GISEL-NEXT: s_mov_b32 s4, exec_lo
+; GFX1132GISEL-NEXT: s_mov_b32 s3, 0
; GFX1132GISEL-NEXT: .LBB6_4: ; =>This Inner Loop Header: Depth=1
-; GFX1132GISEL-NEXT: s_ctz_i32_b32 s3, s2
+; GFX1132GISEL-NEXT: s_ctz_i32_b32 s5, s4
; GFX1132GISEL-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s3
-; GFX1132GISEL-NEXT: s_bitset0_b32 s2, s3
-; GFX1132GISEL-NEXT: s_xor_b32 s1, s1, s6
-; GFX1132GISEL-NEXT: s_cmp_lg_u32 s2, 0
+; GFX1132GISEL-NEXT: v_readlane_b32 s6, v0, s5
+; GFX1132GISEL-NEXT: s_bitset0_b32 s4, s5
+; GFX1132GISEL-NEXT: s_xor_b32 s3, s3, s6
+; GFX1132GISEL-NEXT: s_cmp_lg_u32 s4, 0
; GFX1132GISEL-NEXT: s_cbranch_scc1 .LBB6_4
; GFX1132GISEL-NEXT: ; %bb.5:
-; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s1
+; GFX1132GISEL-NEXT: v_mov_b32_e32 v1, s3
; GFX1132GISEL-NEXT: .LBB6_6: ; %endif
-; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX1132GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1132GISEL-NEXT: s_or_b32 exec_lo, exec_lo, s2
; GFX1132GISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1132GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1132GISEL-NEXT: global_store_b32 v0, v1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
index 7c7d899f7de3e..528c3b07244ce 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.clamp.ll
@@ -1,159 +1,44 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefixes=SI %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga < %s | FileCheck -check-prefixes=VI %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select < %s | FileCheck -check-prefixes=SI-GISEL %s
-; RUN: llc -mtriple=amdgcn -global-isel=1 -new-reg-bank-select -mcpu=tonga < %s | FileCheck -check-prefixes=VI-GISEL %s
+; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI -check-prefix=FUNC %s
declare float @llvm.amdgcn.rsq.clamp.f32(float) #1
declare double @llvm.amdgcn.rsq.clamp.f64(double) #1
+; FUNC-LABEL: {{^}}rsq_clamp_f32:
+; SI: v_rsq_clamp_f32_e32
+
+; VI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x9|0x24}}
+; VI-DAG: v_rsq_f32_e32 [[RSQ:v[0-9]+]], s{{[0-9]+}}
+; VI-DAG: v_min_f32_e32 [[MIN:v[0-9]+]], 0x7f7fffff, [[RSQ]]
+; VI: v_max_f32_e32 [[RESULT:v[0-9]+]], 0xff7fffff, [[MIN]]
+; VI: buffer_store_dword [[RESULT]]
define amdgpu_kernel void @rsq_clamp_f32(ptr addrspace(1) %out, float %src) #0 {
-; SI-LABEL: rsq_clamp_f32:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_rsq_clamp_f32_e32 v0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; SI-NEXT: s_endpgm
-;
-; VI-LABEL: rsq_clamp_f32:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s0, s[4:5], 0x2c
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_rsq_f32_e32 v0, s0
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: v_min_f32_e32 v0, 0x7f7fffff, v0
-; VI-NEXT: v_max_f32_e32 v2, 0xff7fffff, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: s_endpgm
-;
-; SI-GISEL-LABEL: rsq_clamp_f32:
-; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dword s3, s[4:5], 0xb
-; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-GISEL-NEXT: s_mov_b32 s2, -1
-; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: v_rsq_clamp_f32_e32 v0, s3
-; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
-; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; SI-GISEL-NEXT: s_endpgm
-;
-; VI-GISEL-LABEL: rsq_clamp_f32:
-; VI-GISEL: ; %bb.0:
-; VI-GISEL-NEXT: s_load_dword s0, s[4:5], 0x2c
-; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_rsq_f32_e32 v0, s0
-; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-GISEL-NEXT: v_min_f32_e32 v0, 0x7f7fffff, v0
-; VI-GISEL-NEXT: v_max_f32_e32 v2, 0xff7fffff, v0
-; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
-; VI-GISEL-NEXT: s_endpgm
%rsq_clamp = call float @llvm.amdgcn.rsq.clamp.f32(float %src)
store float %rsq_clamp, ptr addrspace(1) %out
ret void
}
+; FUNC-LABEL: {{^}}rsq_clamp_f64:
+; SI: v_rsq_clamp_f64_e32
+
+; TODO: this constant should be folded:
+; VI-DAG: s_mov_b32 [[NEG1:s[0-9]+]], -1
+; VI-DAG: s_mov_b32 s[[LOW1:[0-9]+]], [[NEG1]]
+; VI-DAG: s_mov_b32 s[[HIGH1:[0-9]+]], 0x7fefffff
+; VI-DAG: s_mov_b32 s[[HIGH2:[0-9]+]], 0xffefffff
+; VI-DAG: v_rsq_f64_e32 [[RSQ:v\[[0-9]+:[0-9]+\]]], s[{{[0-9]+:[0-9]+}}
+; VI-DAG: v_min_f64 v[0:1], [[RSQ]], s[[[LOW1]]:[[HIGH1]]]
+; VI-DAG: v_max_f64 v[0:1], v[0:1], s[[[LOW1]]:[[HIGH2]]]
define amdgpu_kernel void @rsq_clamp_f64(ptr addrspace(1) %out, double %src) #0 {
-; SI-LABEL: rsq_clamp_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s4, s0
-; SI-NEXT: s_mov_b32 s5, s1
-; SI-NEXT: v_rsq_clamp_f64_e32 v[0:1], s[2:3]
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
-; SI-NEXT: s_endpgm
-;
-; VI-LABEL: rsq_clamp_f64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_rsq_f64_e32 v[0:1], s[2:3]
-; VI-NEXT: s_mov_b32 s3, 0x7fefffff
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_min_f64 v[0:1], v[0:1], s[2:3]
-; VI-NEXT: s_mov_b32 s3, 0xffefffff
-; VI-NEXT: v_max_f64 v[0:1], v[0:1], s[2:3]
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
-;
-; SI-GISEL-LABEL: rsq_clamp_f64:
-; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: v_rsq_clamp_f64_e32 v[0:1], s[2:3]
-; SI-GISEL-NEXT: s_mov_b32 s2, -1
-; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
-; SI-GISEL-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
-; SI-GISEL-NEXT: s_endpgm
-;
-; VI-GISEL-LABEL: rsq_clamp_f64:
-; VI-GISEL: ; %bb.0:
-; VI-GISEL-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, -1
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, 0x7fefffff
-; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_rsq_f64_e32 v[0:1], s[2:3]
-; VI-GISEL-NEXT: v_min_f64 v[0:1], v[0:1], v[2:3]
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, -1
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, 0xffefffff
-; VI-GISEL-NEXT: v_max_f64 v[0:1], v[0:1], v[2:3]
-; VI-GISEL-NEXT: v_mov_b32_e32 v3, s1
-; VI-GISEL-NEXT: v_mov_b32_e32 v2, s0
-; VI-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-GISEL-NEXT: s_endpgm
%rsq_clamp = call double @llvm.amdgcn.rsq.clamp.f64(double %src)
store double %rsq_clamp, ptr addrspace(1) %out
ret void
}
-; With SDAG, poison folding happens in target specific combine. With GISel this
-; only happens as part of InstCombine so is not tested here.
-
+; FUNC-LABEL: {{^}}rsq_clamp_undef_f32:
+; SI-NOT: v_rsq_clamp_f32
define amdgpu_kernel void @rsq_clamp_undef_f32(ptr addrspace(1) %out) #0 {
-; SI-LABEL: rsq_clamp_undef_f32:
-; SI: ; %bb.0:
-; SI-NEXT: s_endpgm
-;
-; VI-LABEL: rsq_clamp_undef_f32:
-; VI: ; %bb.0:
-; VI-NEXT: s_endpgm
-;
-; SI-GISEL-LABEL: rsq_clamp_undef_f32:
-; SI-GISEL: ; %bb.0:
-; SI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; SI-GISEL-NEXT: v_rsq_clamp_f32_e32 v0, s0
-; SI-GISEL-NEXT: s_mov_b32 s2, -1
-; SI-GISEL-NEXT: s_mov_b32 s3, 0xf000
-; SI-GISEL-NEXT: buffer_store_dword v0, off, s[0:3], 0
-; SI-GISEL-NEXT: s_endpgm
-;
-; VI-GISEL-LABEL: rsq_clamp_undef_f32:
-; VI-GISEL: ; %bb.0:
-; VI-GISEL-NEXT: v_rsq_f32_e32 v0, s0
-; VI-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-GISEL-NEXT: v_min_f32_e32 v0, 0x7f7fffff, v0
-; VI-GISEL-NEXT: v_max_f32_e32 v2, 0xff7fffff, v0
-; VI-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; VI-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; VI-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; VI-GISEL-NEXT: flat_store_dword v[0:1], v2
-; VI-GISEL-NEXT: s_endpgm
%rsq_clamp = call float @llvm.amdgcn.rsq.clamp.f32(float poison)
store float %rsq_clamp, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
index 74b89953aa730..f5481aefa1ec0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.ll
@@ -11,19 +11,19 @@ declare double @llvm.amdgcn.rsq.f64(double) #0
define amdgpu_kernel void @rsq_f32(ptr addrspace(1) %out, float %src) #1 {
; SI-LABEL: rsq_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_rsq_f32_e32 v0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_rsq_f32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: rsq_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_rsq_f32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
index 553ac88dfef89..c55a0d1d5bd95 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.s.barrier.ll
@@ -11,50 +11,49 @@
define amdgpu_kernel void @test_barrier(ptr addrspace(1) %out, i32 %size) #0 {
; VARIANT0-LABEL: test_barrier:
; VARIANT0: ; %bb.0: ; %entry
-; VARIANT0-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; VARIANT0-NEXT: s_load_dword s4, s[4:5], 0xb
-; VARIANT0-NEXT: s_mov_b32 s3, 0xf000
-; VARIANT0-NEXT: s_mov_b32 s2, 0
+; VARIANT0-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; VARIANT0-NEXT: s_mov_b32 s7, 0xf000
+; VARIANT0-NEXT: s_mov_b32 s6, 0
; VARIANT0-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; VARIANT0-NEXT: v_mov_b32_e32 v2, 0
; VARIANT0-NEXT: v_not_b32_e32 v3, v0
; VARIANT0-NEXT: s_waitcnt lgkmcnt(0)
-; VARIANT0-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; VARIANT0-NEXT: s_mov_b64 s[4:5], s[0:1]
+; VARIANT0-NEXT: buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
; VARIANT0-NEXT: s_waitcnt vmcnt(0) expcnt(0)
; VARIANT0-NEXT: s_barrier
-; VARIANT0-NEXT: v_add_i32_e32 v3, vcc, s4, v3
+; VARIANT0-NEXT: v_add_i32_e32 v3, vcc, s2, v3
; VARIANT0-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; VARIANT0-NEXT: v_lshl_b64 v[3:4], v[3:4], 2
-; VARIANT0-NEXT: buffer_load_dword v0, v[3:4], s[0:3], 0 addr64
+; VARIANT0-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
; VARIANT0-NEXT: s_waitcnt vmcnt(0)
-; VARIANT0-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; VARIANT0-NEXT: buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
; VARIANT0-NEXT: s_endpgm
;
; VARIANT1-LABEL: test_barrier:
; VARIANT1: ; %bb.0: ; %entry
-; VARIANT1-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; VARIANT1-NEXT: s_load_dword s4, s[4:5], 0xb
-; VARIANT1-NEXT: s_mov_b32 s3, 0xf000
-; VARIANT1-NEXT: s_mov_b32 s2, 0
+; VARIANT1-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; VARIANT1-NEXT: s_mov_b32 s7, 0xf000
+; VARIANT1-NEXT: s_mov_b32 s6, 0
; VARIANT1-NEXT: v_lshlrev_b32_e32 v1, 2, v0
; VARIANT1-NEXT: v_mov_b32_e32 v2, 0
; VARIANT1-NEXT: v_not_b32_e32 v3, v0
; VARIANT1-NEXT: s_waitcnt lgkmcnt(0)
-; VARIANT1-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; VARIANT1-NEXT: s_mov_b64 s[4:5], s[0:1]
+; VARIANT1-NEXT: buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
; VARIANT1-NEXT: s_barrier
-; VARIANT1-NEXT: v_add_i32_e32 v3, vcc, s4, v3
+; VARIANT1-NEXT: v_add_i32_e32 v3, vcc, s2, v3
; VARIANT1-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; VARIANT1-NEXT: v_lshl_b64 v[3:4], v[3:4], 2
; VARIANT1-NEXT: s_waitcnt expcnt(0)
-; VARIANT1-NEXT: buffer_load_dword v0, v[3:4], s[0:3], 0 addr64
+; VARIANT1-NEXT: buffer_load_dword v0, v[3:4], s[4:7], 0 addr64
; VARIANT1-NEXT: s_waitcnt vmcnt(0)
-; VARIANT1-NEXT: buffer_store_dword v0, v[1:2], s[0:3], 0 addr64
+; VARIANT1-NEXT: buffer_store_dword v0, v[1:2], s[4:7], 0 addr64
; VARIANT1-NEXT: s_endpgm
;
; VARIANT2-LABEL: test_barrier:
; VARIANT2: ; %bb.0: ; %entry
-; VARIANT2-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VARIANT2-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VARIANT2-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VARIANT2-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VARIANT2-NEXT: s_waitcnt lgkmcnt(0)
; VARIANT2-NEXT: global_store_dword v2, v0, s[0:1]
@@ -95,8 +94,7 @@ define amdgpu_kernel void @test_barrier(ptr addrspace(1) %out, i32 %size) #0 {
;
; VARIANT3-LABEL: test_barrier:
; VARIANT3: ; %bb.0: ; %entry
-; VARIANT3-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VARIANT3-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VARIANT3-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VARIANT3-NEXT: v_lshlrev_b32_e32 v2, 2, v0
; VARIANT3-NEXT: s_waitcnt lgkmcnt(0)
; VARIANT3-NEXT: global_store_dword v2, v0, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll
index ce03681f2eca8..d06b16c7afb2c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.set.inactive.ll
@@ -5,17 +5,18 @@
define amdgpu_kernel void @set_inactive(ptr addrspace(1) %out, i32 %in) {
; GCN-LABEL: set_inactive:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 42, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 42, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call i32 @llvm.amdgcn.set.inactive.i32(i32 %in, i32 42) #0
%tmp = call i32 @llvm.amdgcn.strict.wwm.i32(i32 %tmp.0)
@@ -89,17 +90,15 @@ define amdgpu_kernel void @set_inactive_imm_poison_64(ptr addrspace(1) %out) {
define amdgpu_kernel void @set_inactive_scc(ptr addrspace(1) %out, i32 %in, <4 x i32> inreg %desc) {
; GCN-LABEL: set_inactive_scc:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_buffer_load_dword s7, s[0:3], 0x0
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: v_mov_b32_e32 v1, s6
+; GCN-NEXT: s_buffer_load_dword s4, s[4:7], 0x0
+; GCN-NEXT: v_mov_b32_e32 v1, s2
; GCN-NEXT: s_or_saveexec_b64 s[2:3], -1
; GCN-NEXT: v_cndmask_b32_e64 v0, 42, v1, s[2:3]
; GCN-NEXT: s_mov_b64 exec, s[2:3]
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_lg_u32 s7, 56
+; GCN-NEXT: s_cmp_lg_u32 s4, 56
; GCN-NEXT: v_mov_b32_e32 v1, v0
; GCN-NEXT: s_mov_b64 s[2:3], -1
; GCN-NEXT: s_cbranch_scc1 .LBB4_3
@@ -141,18 +140,19 @@ define amdgpu_kernel void @set_inactive_scc(ptr addrspace(1) %out, i32 %in, <4 x
define amdgpu_kernel void @set_inactive_f32(ptr addrspace(1) %out, float %in) {
; GCN-LABEL: set_inactive_f32:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
; GCN-NEXT: v_mov_b32_e32 v0, 0x40400000
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call float @llvm.amdgcn.set.inactive.f32(float %in, float 3.0) #0
%tmp = call float @llvm.amdgcn.strict.wwm.f32(float %tmp.0)
@@ -192,18 +192,19 @@ define amdgpu_kernel void @set_inactive_f64(ptr addrspace(1) %out, double %in) {
define amdgpu_kernel void @set_inactive_v2i16(ptr addrspace(1) %out, <2 x i16> %in) {
; GCN-LABEL: set_inactive_v2i16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
; GCN-NEXT: v_mov_b32_e32 v0, 0x10001
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call <2 x i16> @llvm.amdgcn.set.inactive.v2i16(<2 x i16> %in, <2 x i16> <i16 1, i16 1>) #0
%tmp = call <2 x i16> @llvm.amdgcn.strict.wwm.v2i16(<2 x i16> %tmp.0)
@@ -214,18 +215,19 @@ define amdgpu_kernel void @set_inactive_v2i16(ptr addrspace(1) %out, <2 x i16> %
define amdgpu_kernel void @set_inactive_v2f16(ptr addrspace(1) %out, <2 x half> %in) {
; GCN-LABEL: set_inactive_v2f16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
; GCN-NEXT: v_mov_b32_e32 v0, 0x3c003c00
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call <2 x half> @llvm.amdgcn.set.inactive.v2f16(<2 x half> %in, <2 x half> <half 1.0, half 1.0>) #0
%tmp = call <2 x half> @llvm.amdgcn.strict.wwm.v2i16(<2 x half> %tmp.0)
@@ -290,18 +292,19 @@ define amdgpu_kernel void @set_inactive_v2f32(ptr addrspace(1) %out, <2 x float>
define amdgpu_kernel void @set_inactive_v2bf16(ptr addrspace(1) %out, <2 x bfloat> %in) {
; GCN-LABEL: set_inactive_v2bf16:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
; GCN-NEXT: v_mov_b32_e32 v0, 0x3f803f80
-; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: v_cndmask_b32_e64 v0, v0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call <2 x bfloat> @llvm.amdgcn.set.inactive.v2bf16(<2 x bfloat> %in, <2 x bfloat> <bfloat 1.0, bfloat 1.0>) #0
%tmp = call <2 x bfloat> @llvm.amdgcn.strict.wwm.v2bf16(<2 x bfloat> %tmp.0)
@@ -423,17 +426,18 @@ define amdgpu_kernel void @set_inactive_p0(ptr addrspace(1) %out, ptr %in) {
define amdgpu_kernel void @set_inactive_p2(ptr addrspace(1) %out, ptr addrspace(2) %in) {
; GCN-LABEL: set_inactive_p2:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call ptr addrspace(2) @llvm.amdgcn.set.inactive.p2(ptr addrspace(2) %in, ptr addrspace(2) zeroinitializer) #0
%tmp = call ptr addrspace(2) @llvm.amdgcn.strict.wwm.p2(ptr addrspace(2) %tmp.0)
@@ -444,17 +448,18 @@ define amdgpu_kernel void @set_inactive_p2(ptr addrspace(1) %out, ptr addrspace(
define amdgpu_kernel void @set_inactive_p3(ptr addrspace(1) %out, ptr addrspace(3) %in) {
; GCN-LABEL: set_inactive_p3:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call ptr addrspace(3) @llvm.amdgcn.set.inactive.p3(ptr addrspace(3) %in, ptr addrspace(3) zeroinitializer) #0
%tmp = call ptr addrspace(3) @llvm.amdgcn.strict.wwm.p3(ptr addrspace(3) %tmp.0)
@@ -465,17 +470,18 @@ define amdgpu_kernel void @set_inactive_p3(ptr addrspace(1) %out, ptr addrspace(
define amdgpu_kernel void @set_inactive_p5(ptr addrspace(1) %out, ptr addrspace(5) %in) {
; GCN-LABEL: set_inactive_p5:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call ptr addrspace(5) @llvm.amdgcn.set.inactive.p5(ptr addrspace(5) %in, ptr addrspace(5) zeroinitializer) #0
%tmp = call ptr addrspace(5) @llvm.amdgcn.strict.wwm.p5(ptr addrspace(5) %tmp.0)
@@ -486,17 +492,18 @@ define amdgpu_kernel void @set_inactive_p5(ptr addrspace(1) %out, ptr addrspace(
define amdgpu_kernel void @set_inactive_p6(ptr addrspace(1) %out, ptr addrspace(6) %in) {
; GCN-LABEL: set_inactive_p6:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v1, s6
-; GCN-NEXT: s_or_saveexec_b64 s[4:5], -1
-; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[4:5]
-; GCN-NEXT: s_mov_b64 exec, s[4:5]
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: s_or_saveexec_b64 s[0:1], -1
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, v1, s[0:1]
+; GCN-NEXT: s_mov_b64 exec, s[0:1]
; GCN-NEXT: v_mov_b32_e32 v1, v0
-; GCN-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v1, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tmp.0 = call ptr addrspace(6) @llvm.amdgcn.set.inactive.p6(ptr addrspace(6) %in, ptr addrspace(6) null) #0
%tmp = call ptr addrspace(6) @llvm.amdgcn.strict.wwm.p6(ptr addrspace(6) %tmp.0)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.format.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.format.d16.ll
index ff421d1f22699..85da8083a2189 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.format.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.store.format.d16.ll
@@ -50,9 +50,7 @@ main_body:
define amdgpu_kernel void @buffer_store_format_d16_xyz(<4 x i32> %rsrc, <4 x half> %data, i32 %index) {
; UNPACKED-LABEL: buffer_store_format_d16_xyz:
; UNPACKED: ; %bb.0: ; %main_body
-; UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
; UNPACKED-NEXT: s_lshr_b32 s7, s4, 16
@@ -66,9 +64,7 @@ define amdgpu_kernel void @buffer_store_format_d16_xyz(<4 x i32> %rsrc, <4 x hal
;
; PACKED-LABEL: buffer_store_format_d16_xyz:
; PACKED: ; %bb.0: ; %main_body
-; PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-NEXT: s_and_b32 s5, s5, 0xffff
; PACKED-NEXT: v_mov_b32_e32 v0, s4
@@ -85,9 +81,7 @@ main_body:
define amdgpu_kernel void @buffer_store_format_d16_xyzw(<4 x i32> %rsrc, <4 x half> %data, i32 %index) {
; UNPACKED-LABEL: buffer_store_format_d16_xyzw:
; UNPACKED: ; %bb.0: ; %main_body
-; UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; UNPACKED-NEXT: s_lshr_b32 s7, s5, 16
; UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
@@ -103,9 +97,7 @@ define amdgpu_kernel void @buffer_store_format_d16_xyzw(<4 x i32> %rsrc, <4 x ha
;
; PACKED-LABEL: buffer_store_format_d16_xyzw:
; PACKED: ; %bb.0: ; %main_body
-; PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-NEXT: v_mov_b32_e32 v0, s4
; PACKED-NEXT: v_mov_b32_e32 v1, s5
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.store.format.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.store.format.d16.ll
index 8fea08df92f36..8f5a10e20517b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.store.format.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.store.format.d16.ll
@@ -50,9 +50,7 @@ main_body:
define amdgpu_kernel void @buffer_store_format_d16_xyz(ptr addrspace(8) %rsrc, <4 x half> %data, i32 %index) {
; UNPACKED-LABEL: buffer_store_format_d16_xyz:
; UNPACKED: ; %bb.0: ; %main_body
-; UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
; UNPACKED-NEXT: s_lshr_b32 s7, s4, 16
@@ -66,9 +64,7 @@ define amdgpu_kernel void @buffer_store_format_d16_xyz(ptr addrspace(8) %rsrc, <
;
; PACKED-LABEL: buffer_store_format_d16_xyz:
; PACKED: ; %bb.0: ; %main_body
-; PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-NEXT: s_and_b32 s5, s5, 0xffff
; PACKED-NEXT: v_mov_b32_e32 v0, s4
@@ -85,9 +81,7 @@ main_body:
define amdgpu_kernel void @buffer_store_format_d16_xyzw(ptr addrspace(8) %rsrc, <4 x half> %data, i32 %index) {
; UNPACKED-LABEL: buffer_store_format_d16_xyzw:
; UNPACKED: ; %bb.0: ; %main_body
-; UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; UNPACKED-NEXT: s_lshr_b32 s7, s5, 16
; UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
@@ -103,9 +97,7 @@ define amdgpu_kernel void @buffer_store_format_d16_xyzw(ptr addrspace(8) %rsrc,
;
; PACKED-LABEL: buffer_store_format_d16_xyzw:
; PACKED: ; %bb.0: ; %main_body
-; PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-NEXT: v_mov_b32_e32 v0, s4
; PACKED-NEXT: v_mov_b32_e32 v1, s5
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll
index eb28f63e90803..9007e719ee686 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.tbuffer.store.d16.ll
@@ -106,9 +106,7 @@ main_body:
define amdgpu_kernel void @tbuffer_store_d16_xyz(ptr addrspace(8) %rsrc, <4 x half> %data, i32 %vindex) {
; PREGFX10-UNPACKED-LABEL: tbuffer_store_d16_xyz:
; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
-; PREGFX10-UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PREGFX10-UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; PREGFX10-UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; PREGFX10-UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PREGFX10-UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; PREGFX10-UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
; PREGFX10-UNPACKED-NEXT: s_lshr_b32 s7, s4, 16
@@ -122,9 +120,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyz(ptr addrspace(8) %rsrc, <4 x ha
;
; PREGFX10-PACKED-LABEL: tbuffer_store_d16_xyz:
; PREGFX10-PACKED: ; %bb.0: ; %main_body
-; PREGFX10-PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PREGFX10-PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PREGFX10-PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PREGFX10-PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PREGFX10-PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PREGFX10-PACKED-NEXT: s_and_b32 s5, s5, 0xffff
; PREGFX10-PACKED-NEXT: v_mov_b32_e32 v0, s4
@@ -135,10 +131,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyz(ptr addrspace(8) %rsrc, <4 x ha
;
; GFX10-PACKED-LABEL: tbuffer_store_d16_xyz:
; GFX10-PACKED: ; %bb.0: ; %main_body
-; GFX10-PACKED-NEXT: s_clause 0x2
-; GFX10-PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX10-PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; GFX10-PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX10-PACKED-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-PACKED-NEXT: s_and_b32 s5, s5, 0xffff
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, s4
@@ -149,15 +142,12 @@ define amdgpu_kernel void @tbuffer_store_d16_xyz(ptr addrspace(8) %rsrc, <4 x ha
;
; GFX11-PACKED-LABEL: tbuffer_store_d16_xyz:
; GFX11-PACKED: ; %bb.0: ; %main_body
-; GFX11-PACKED-NEXT: s_clause 0x2
-; GFX11-PACKED-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
-; GFX11-PACKED-NEXT: s_load_b32 s8, s[4:5], 0x18
-; GFX11-PACKED-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-PACKED-NEXT: s_load_b256 s[0:7], s[4:5], 0x0
; GFX11-PACKED-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-PACKED-NEXT: s_and_b32 s4, s7, 0xffff
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s6
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v2, s8
+; GFX11-PACKED-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v1, s5
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v2, s6
; GFX11-PACKED-NEXT: tbuffer_store_d16_format_xyz v[0:1], v2, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] idxen
; GFX11-PACKED-NEXT: s_endpgm
main_body:
@@ -169,9 +159,7 @@ main_body:
define amdgpu_kernel void @tbuffer_store_d16_xyzw(ptr addrspace(8) %rsrc, <4 x half> %data, i32 %vindex) {
; PREGFX10-UNPACKED-LABEL: tbuffer_store_d16_xyzw:
; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
-; PREGFX10-UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PREGFX10-UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; PREGFX10-UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; PREGFX10-UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PREGFX10-UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; PREGFX10-UNPACKED-NEXT: s_lshr_b32 s7, s5, 16
; PREGFX10-UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
@@ -187,9 +175,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyzw(ptr addrspace(8) %rsrc, <4 x h
;
; PREGFX10-PACKED-LABEL: tbuffer_store_d16_xyzw:
; PREGFX10-PACKED: ; %bb.0: ; %main_body
-; PREGFX10-PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PREGFX10-PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PREGFX10-PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PREGFX10-PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PREGFX10-PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PREGFX10-PACKED-NEXT: v_mov_b32_e32 v0, s4
; PREGFX10-PACKED-NEXT: v_mov_b32_e32 v1, s5
@@ -199,10 +185,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyzw(ptr addrspace(8) %rsrc, <4 x h
;
; GFX10-PACKED-LABEL: tbuffer_store_d16_xyzw:
; GFX10-PACKED: ; %bb.0: ; %main_body
-; GFX10-PACKED-NEXT: s_clause 0x2
-; GFX10-PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX10-PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; GFX10-PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX10-PACKED-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, s4
; GFX10-PACKED-NEXT: v_mov_b32_e32 v1, s5
@@ -212,14 +195,11 @@ define amdgpu_kernel void @tbuffer_store_d16_xyzw(ptr addrspace(8) %rsrc, <4 x h
;
; GFX11-PACKED-LABEL: tbuffer_store_d16_xyzw:
; GFX11-PACKED: ; %bb.0: ; %main_body
-; GFX11-PACKED-NEXT: s_clause 0x2
-; GFX11-PACKED-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
-; GFX11-PACKED-NEXT: s_load_b32 s8, s[4:5], 0x18
-; GFX11-PACKED-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-PACKED-NEXT: s_load_b256 s[0:7], s[4:5], 0x0
; GFX11-PACKED-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s6
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v1, s7
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v2, s8
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v1, s5
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v2, s6
; GFX11-PACKED-NEXT: tbuffer_store_d16_format_xyzw v[0:1], v2, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] idxen
; GFX11-PACKED-NEXT: s_endpgm
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
index 21f179419d5d1..b959397a1aaba 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.tbuffer.store.d16.ll
@@ -132,9 +132,7 @@ main_body:
define amdgpu_kernel void @tbuffer_store_d16_xyz(<4 x i32> %rsrc, <4 x half> %data, i32 %vindex) {
; PREGFX10-UNPACKED-LABEL: tbuffer_store_d16_xyz:
; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
-; PREGFX10-UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PREGFX10-UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; PREGFX10-UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; PREGFX10-UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PREGFX10-UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; PREGFX10-UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
; PREGFX10-UNPACKED-NEXT: s_lshr_b32 s7, s4, 16
@@ -148,9 +146,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyz(<4 x i32> %rsrc, <4 x half> %da
;
; PREGFX10-PACKED-LABEL: tbuffer_store_d16_xyz:
; PREGFX10-PACKED: ; %bb.0: ; %main_body
-; PREGFX10-PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PREGFX10-PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PREGFX10-PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PREGFX10-PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PREGFX10-PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PREGFX10-PACKED-NEXT: s_and_b32 s5, s5, 0xffff
; PREGFX10-PACKED-NEXT: v_mov_b32_e32 v0, s4
@@ -161,10 +157,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyz(<4 x i32> %rsrc, <4 x half> %da
;
; GFX10-PACKED-LABEL: tbuffer_store_d16_xyz:
; GFX10-PACKED: ; %bb.0: ; %main_body
-; GFX10-PACKED-NEXT: s_clause 0x2
-; GFX10-PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX10-PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; GFX10-PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX10-PACKED-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-PACKED-NEXT: s_and_b32 s5, s5, 0xffff
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, s4
@@ -175,15 +168,12 @@ define amdgpu_kernel void @tbuffer_store_d16_xyz(<4 x i32> %rsrc, <4 x half> %da
;
; GFX11-PACKED-LABEL: tbuffer_store_d16_xyz:
; GFX11-PACKED: ; %bb.0: ; %main_body
-; GFX11-PACKED-NEXT: s_clause 0x2
-; GFX11-PACKED-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
-; GFX11-PACKED-NEXT: s_load_b32 s8, s[4:5], 0x18
-; GFX11-PACKED-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-PACKED-NEXT: s_load_b256 s[0:7], s[4:5], 0x0
; GFX11-PACKED-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-PACKED-NEXT: s_and_b32 s4, s7, 0xffff
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s6
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v1, s4
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v2, s8
+; GFX11-PACKED-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v1, s5
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v2, s6
; GFX11-PACKED-NEXT: tbuffer_store_d16_format_xyz v[0:1], v2, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] idxen
; GFX11-PACKED-NEXT: s_endpgm
;
@@ -220,9 +210,7 @@ main_body:
define amdgpu_kernel void @tbuffer_store_d16_xyzw(<4 x i32> %rsrc, <4 x half> %data, i32 %vindex) {
; PREGFX10-UNPACKED-LABEL: tbuffer_store_d16_xyzw:
; PREGFX10-UNPACKED: ; %bb.0: ; %main_body
-; PREGFX10-UNPACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PREGFX10-UNPACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; PREGFX10-UNPACKED-NEXT: s_load_dword s6, s[8:9], 0x18
+; PREGFX10-UNPACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PREGFX10-UNPACKED-NEXT: s_waitcnt lgkmcnt(0)
; PREGFX10-UNPACKED-NEXT: s_lshr_b32 s7, s5, 16
; PREGFX10-UNPACKED-NEXT: s_and_b32 s5, s5, 0xffff
@@ -238,9 +226,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyzw(<4 x i32> %rsrc, <4 x half> %d
;
; PREGFX10-PACKED-LABEL: tbuffer_store_d16_xyzw:
; PREGFX10-PACKED: ; %bb.0: ; %main_body
-; PREGFX10-PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; PREGFX10-PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; PREGFX10-PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; PREGFX10-PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; PREGFX10-PACKED-NEXT: s_waitcnt lgkmcnt(0)
; PREGFX10-PACKED-NEXT: v_mov_b32_e32 v0, s4
; PREGFX10-PACKED-NEXT: v_mov_b32_e32 v1, s5
@@ -250,10 +236,7 @@ define amdgpu_kernel void @tbuffer_store_d16_xyzw(<4 x i32> %rsrc, <4 x half> %d
;
; GFX10-PACKED-LABEL: tbuffer_store_d16_xyzw:
; GFX10-PACKED: ; %bb.0: ; %main_body
-; GFX10-PACKED-NEXT: s_clause 0x2
-; GFX10-PACKED-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX10-PACKED-NEXT: s_load_dword s6, s[8:9], 0x18
-; GFX10-PACKED-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-PACKED-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX10-PACKED-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-PACKED-NEXT: v_mov_b32_e32 v0, s4
; GFX10-PACKED-NEXT: v_mov_b32_e32 v1, s5
@@ -263,14 +246,11 @@ define amdgpu_kernel void @tbuffer_store_d16_xyzw(<4 x i32> %rsrc, <4 x half> %d
;
; GFX11-PACKED-LABEL: tbuffer_store_d16_xyzw:
; GFX11-PACKED: ; %bb.0: ; %main_body
-; GFX11-PACKED-NEXT: s_clause 0x2
-; GFX11-PACKED-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
-; GFX11-PACKED-NEXT: s_load_b32 s8, s[4:5], 0x18
-; GFX11-PACKED-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-PACKED-NEXT: s_load_b256 s[0:7], s[4:5], 0x0
; GFX11-PACKED-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s6
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v1, s7
-; GFX11-PACKED-NEXT: v_mov_b32_e32 v2, s8
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v0, s4
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v1, s5
+; GFX11-PACKED-NEXT: v_mov_b32_e32 v2, s6
; GFX11-PACKED-NEXT: tbuffer_store_d16_format_xyzw v[0:1], v2, s[0:3], 0 format:[BUF_FMT_10_10_10_2_SNORM] idxen
; GFX11-PACKED-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ubfe.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ubfe.ll
index e421e2c8ebfc4..b8b8f6ac326dc 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ubfe.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ubfe.ll
@@ -137,21 +137,21 @@ define amdgpu_kernel void @bfe_u32_imm_arg_arg(ptr addrspace(1) %out, i32 %src1,
define amdgpu_kernel void @bfe_u32_arg_0_width_reg_offset(ptr addrspace(1) %out, i32 %src0, i32 %src1) #0 {
; SI-LABEL: bfe_u32_arg_0_width_reg_offset:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: bfe_u32_arg_0_width_reg_offset:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_endpgm
%bfe_u32 = call i32 @llvm.amdgcn.ubfe.i32(i32 %src0, i32 %src1, i32 0)
@@ -162,21 +162,21 @@ define amdgpu_kernel void @bfe_u32_arg_0_width_reg_offset(ptr addrspace(1) %out,
define amdgpu_kernel void @bfe_u32_arg_0_width_imm_offset(ptr addrspace(1) %out, i32 %src0, i32 %src1) #0 {
; SI-LABEL: bfe_u32_arg_0_width_imm_offset:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: bfe_u32_arg_0_width_imm_offset:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_endpgm
%bfe_u32 = call i32 @llvm.amdgcn.ubfe.i32(i32 %src0, i32 8, i32 0)
@@ -1581,10 +1581,9 @@ define amdgpu_kernel void @simplify_bfe_u32_multi_use_arg(ptr addrspace(1) %out0
define amdgpu_kernel void @lshr_and(ptr addrspace(1) %out, i32 %a) #0 {
; SI-LABEL: lshr_and:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_bfe_u32 s4, s2, 0x30006
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -1593,14 +1592,15 @@ define amdgpu_kernel void @lshr_and(ptr addrspace(1) %out, i32 %a) #0 {
;
; VI-LABEL: lshr_and:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfe_u32 s4, s6, 0x30006
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_bfe_u32 s0, s2, 0x30006
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
%b = lshr i32 %a, 6
%c = and i32 %b, 7
@@ -1612,27 +1612,29 @@ define amdgpu_kernel void @v_lshr_and(ptr addrspace(1) %out, i32 %a, i32 %b) #0
; SI-LABEL: v_lshr_and:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_lshr_b32 s2, s4, s5
-; SI-NEXT: s_and_b32 s4, s2, 7
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_lshr_b32 s2, s2, s3
+; SI-NEXT: s_and_b32 s2, s2, 7
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v_lshr_and:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: s_lshr_b32 s4, s4, s5
-; VI-NEXT: s_and_b32 s4, s4, 7
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_lshr_b32 s0, s2, s3
+; VI-NEXT: s_and_b32 s0, s0, 7
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
%c = lshr i32 %a, %b
%d = and i32 %c, 7
@@ -1643,10 +1645,9 @@ define amdgpu_kernel void @v_lshr_and(ptr addrspace(1) %out, i32 %a, i32 %b) #0
define amdgpu_kernel void @and_lshr(ptr addrspace(1) %out, i32 %a) #0 {
; SI-LABEL: and_lshr:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_bfe_u32 s4, s2, 0x30006
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -1655,14 +1656,15 @@ define amdgpu_kernel void @and_lshr(ptr addrspace(1) %out, i32 %a) #0 {
;
; VI-LABEL: and_lshr:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfe_u32 s4, s6, 0x30006
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_bfe_u32 s0, s2, 0x30006
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
%b = and i32 %a, 448
%c = lshr i32 %b, 6
@@ -1673,10 +1675,9 @@ define amdgpu_kernel void @and_lshr(ptr addrspace(1) %out, i32 %a) #0 {
define amdgpu_kernel void @and_lshr2(ptr addrspace(1) %out, i32 %a) #0 {
; SI-LABEL: and_lshr2:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_bfe_u32 s4, s2, 0x30006
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -1685,14 +1686,15 @@ define amdgpu_kernel void @and_lshr2(ptr addrspace(1) %out, i32 %a) #0 {
;
; VI-LABEL: and_lshr2:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfe_u32 s4, s6, 0x30006
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_bfe_u32 s0, s2, 0x30006
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
%b = and i32 %a, 511
%c = lshr i32 %b, 6
@@ -1703,10 +1705,9 @@ define amdgpu_kernel void @and_lshr2(ptr addrspace(1) %out, i32 %a) #0 {
define amdgpu_kernel void @shl_lshr(ptr addrspace(1) %out, i32 %a) #0 {
; SI-LABEL: shl_lshr:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_bfe_u32 s4, s2, 0x150002
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, s4
@@ -1715,14 +1716,15 @@ define amdgpu_kernel void @shl_lshr(ptr addrspace(1) %out, i32 %a) #0 {
;
; VI-LABEL: shl_lshr:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfe_u32 s4, s6, 0x150002
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_bfe_u32 s0, s2, 0x150002
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
%b = shl i32 %a, 9
%c = lshr i32 %b, 11
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
index a5189acba98c0..b421fab6a386a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.writelane.ll
@@ -322,8 +322,7 @@ define amdgpu_kernel void @test_writelane_sreg_f64(ptr addrspace(1) %out, double
define amdgpu_kernel void @test_writelane_imm_sreg_i32(ptr addrspace(1) %out, i32 %src1) #1 {
; GFX802-SDAG-LABEL: test_writelane_imm_sreg_i32:
; GFX802-SDAG: ; %bb.0:
-; GFX802-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX802-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX802-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX802-SDAG-NEXT: s_add_i32 s12, s12, s17
; GFX802-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -339,9 +338,7 @@ define amdgpu_kernel void @test_writelane_imm_sreg_i32(ptr addrspace(1) %out, i3
;
; GFX1010-SDAG-LABEL: test_writelane_imm_sreg_i32:
; GFX1010-SDAG: ; %bb.0:
-; GFX1010-SDAG-NEXT: s_clause 0x1
-; GFX1010-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX1010-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX1010-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX1010-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1010-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1010-SDAG-NEXT: s_load_dword s3, s[0:1], 0x0
@@ -353,9 +350,7 @@ define amdgpu_kernel void @test_writelane_imm_sreg_i32(ptr addrspace(1) %out, i3
;
; GFX1100-SDAG-LABEL: test_writelane_imm_sreg_i32:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_clause 0x1
-; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX1100-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: s_load_b32 s3, s[0:1], 0x0
@@ -367,8 +362,7 @@ define amdgpu_kernel void @test_writelane_imm_sreg_i32(ptr addrspace(1) %out, i3
;
; GFX802-GISEL-LABEL: test_writelane_imm_sreg_i32:
; GFX802-GISEL: ; %bb.0:
-; GFX802-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX802-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX802-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX802-GISEL-NEXT: s_add_i32 s12, s12, s17
; GFX802-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -384,9 +378,7 @@ define amdgpu_kernel void @test_writelane_imm_sreg_i32(ptr addrspace(1) %out, i3
;
; GFX1010-GISEL-LABEL: test_writelane_imm_sreg_i32:
; GFX1010-GISEL: ; %bb.0:
-; GFX1010-GISEL-NEXT: s_clause 0x1
-; GFX1010-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX1010-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX1010-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX1010-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1010-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1010-GISEL-NEXT: s_load_dword s3, s[0:1], 0x0
@@ -398,9 +390,7 @@ define amdgpu_kernel void @test_writelane_imm_sreg_i32(ptr addrspace(1) %out, i3
;
; GFX1100-GISEL-LABEL: test_writelane_imm_sreg_i32:
; GFX1100-GISEL: ; %bb.0:
-; GFX1100-GISEL-NEXT: s_clause 0x1
-; GFX1100-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX1100-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX1100-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX1100-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1100-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x0
@@ -418,103 +408,93 @@ define amdgpu_kernel void @test_writelane_imm_sreg_i32(ptr addrspace(1) %out, i3
define amdgpu_kernel void @test_writelane_imm_sreg_i64(ptr addrspace(1) %out, i32 %src1) #1 {
; GFX802-SDAG-LABEL: test_writelane_imm_sreg_i64:
; GFX802-SDAG: ; %bb.0:
-; GFX802-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX802-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX802-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX802-SDAG-NEXT: s_add_i32 s12, s12, s17
; GFX802-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX802-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX802-SDAG-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX802-SDAG-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX802-SDAG-NEXT: v_mov_b32_e32 v3, s1
; GFX802-SDAG-NEXT: v_mov_b32_e32 v2, s0
; GFX802-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX802-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX802-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX802-SDAG-NEXT: v_writelane_b32 v1, 0, s4
-; GFX802-SDAG-NEXT: v_writelane_b32 v0, 32, s4
+; GFX802-SDAG-NEXT: v_mov_b32_e32 v1, s5
+; GFX802-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX802-SDAG-NEXT: v_writelane_b32 v1, 0, s2
+; GFX802-SDAG-NEXT: v_writelane_b32 v0, 32, s2
; GFX802-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX802-SDAG-NEXT: s_endpgm
;
; GFX1010-SDAG-LABEL: test_writelane_imm_sreg_i64:
; GFX1010-SDAG: ; %bb.0:
-; GFX1010-SDAG-NEXT: s_clause 0x1
-; GFX1010-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX1010-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX1010-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX1010-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1010-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1010-SDAG-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1010-SDAG-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1010-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1010-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX1010-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX1010-SDAG-NEXT: v_writelane_b32 v1, 0, s4
-; GFX1010-SDAG-NEXT: v_writelane_b32 v0, 32, s4
+; GFX1010-SDAG-NEXT: v_mov_b32_e32 v1, s5
+; GFX1010-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX1010-SDAG-NEXT: v_writelane_b32 v1, 0, s2
+; GFX1010-SDAG-NEXT: v_writelane_b32 v0, 32, s2
; GFX1010-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX1010-SDAG-NEXT: s_endpgm
;
; GFX1100-SDAG-LABEL: test_writelane_imm_sreg_i64:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_clause 0x1
-; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX1100-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x8
+; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1100-SDAG-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX1100-SDAG-NEXT: v_writelane_b32 v1, 0, s4
-; GFX1100-SDAG-NEXT: v_writelane_b32 v0, 32, s4
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, s5
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX1100-SDAG-NEXT: v_writelane_b32 v1, 0, s2
+; GFX1100-SDAG-NEXT: v_writelane_b32 v0, 32, s2
; GFX1100-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1100-SDAG-NEXT: s_endpgm
;
; GFX802-GISEL-LABEL: test_writelane_imm_sreg_i64:
; GFX802-GISEL: ; %bb.0:
-; GFX802-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX802-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX802-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX802-GISEL-NEXT: s_add_i32 s12, s12, s17
; GFX802-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX802-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX802-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX802-GISEL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX802-GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX802-GISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX802-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX802-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX802-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX802-GISEL-NEXT: v_writelane_b32 v0, 32, s4
-; GFX802-GISEL-NEXT: v_writelane_b32 v1, 0, s4
+; GFX802-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX802-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX802-GISEL-NEXT: v_writelane_b32 v0, 32, s2
+; GFX802-GISEL-NEXT: v_writelane_b32 v1, 0, s2
; GFX802-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX802-GISEL-NEXT: s_endpgm
;
; GFX1010-GISEL-LABEL: test_writelane_imm_sreg_i64:
; GFX1010-GISEL: ; %bb.0:
-; GFX1010-GISEL-NEXT: s_clause 0x1
-; GFX1010-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX1010-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX1010-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX1010-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1010-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1010-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1010-GISEL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1010-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1010-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX1010-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX1010-GISEL-NEXT: v_writelane_b32 v0, 32, s4
-; GFX1010-GISEL-NEXT: v_writelane_b32 v1, 0, s4
+; GFX1010-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX1010-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX1010-GISEL-NEXT: v_writelane_b32 v0, 32, s2
+; GFX1010-GISEL-NEXT: v_writelane_b32 v1, 0, s2
; GFX1010-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX1010-GISEL-NEXT: s_endpgm
;
; GFX1100-GISEL-LABEL: test_writelane_imm_sreg_i64:
; GFX1100-GISEL: ; %bb.0:
-; GFX1100-GISEL-NEXT: s_clause 0x1
-; GFX1100-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX1100-GISEL-NEXT: s_load_b32 s4, s[4:5], 0x8
+; GFX1100-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX1100-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1100-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-GISEL-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1100-GISEL-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1100-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX1100-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX1100-GISEL-NEXT: v_writelane_b32 v0, 32, s4
-; GFX1100-GISEL-NEXT: v_writelane_b32 v1, 0, s4
+; GFX1100-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX1100-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX1100-GISEL-NEXT: v_writelane_b32 v0, 32, s2
+; GFX1100-GISEL-NEXT: v_writelane_b32 v1, 0, s2
; GFX1100-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1100-GISEL-NEXT: s_endpgm
%oldval = load i64, ptr addrspace(1) %out
@@ -526,112 +506,101 @@ define amdgpu_kernel void @test_writelane_imm_sreg_i64(ptr addrspace(1) %out, i3
define amdgpu_kernel void @test_writelane_imm_sreg_f64(ptr addrspace(1) %out, i32 %src1) #1 {
; GFX802-SDAG-LABEL: test_writelane_imm_sreg_f64:
; GFX802-SDAG: ; %bb.0:
-; GFX802-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX802-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX802-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX802-SDAG-NEXT: s_add_i32 s12, s12, s17
-; GFX802-SDAG-NEXT: s_mov_b32 s5, 0x40400000
-; GFX802-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX802-SDAG-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX802-SDAG-NEXT: s_mov_b32 m0, s4
-; GFX802-SDAG-NEXT: v_mov_b32_e32 v3, s1
+; GFX802-SDAG-NEXT: s_mov_b32 s3, 0x40400000
+; GFX802-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX802-SDAG-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX802-SDAG-NEXT: s_mov_b32 m0, s2
+; GFX802-SDAG-NEXT: v_mov_b32_e32 v3, s1
; GFX802-SDAG-NEXT: v_mov_b32_e32 v2, s0
; GFX802-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX802-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX802-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX802-SDAG-NEXT: v_writelane_b32 v1, s5, m0
-; GFX802-SDAG-NEXT: v_writelane_b32 v0, 0, s4
+; GFX802-SDAG-NEXT: v_mov_b32_e32 v1, s5
+; GFX802-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX802-SDAG-NEXT: v_writelane_b32 v1, s3, m0
+; GFX802-SDAG-NEXT: v_writelane_b32 v0, 0, s2
; GFX802-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX802-SDAG-NEXT: s_endpgm
;
; GFX1010-SDAG-LABEL: test_writelane_imm_sreg_f64:
; GFX1010-SDAG: ; %bb.0:
-; GFX1010-SDAG-NEXT: s_clause 0x1
-; GFX1010-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX1010-SDAG-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX1010-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1010-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX1010-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1010-SDAG-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1010-SDAG-NEXT: s_mov_b32 s3, 0x40400000
+; GFX1010-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1010-SDAG-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1010-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1010-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX1010-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX1010-SDAG-NEXT: s_mov_b32 s2, 0x40400000
-; GFX1010-SDAG-NEXT: v_writelane_b32 v1, s2, s4
-; GFX1010-SDAG-NEXT: v_writelane_b32 v0, 0, s4
+; GFX1010-SDAG-NEXT: v_mov_b32_e32 v1, s5
+; GFX1010-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX1010-SDAG-NEXT: v_writelane_b32 v1, s3, s2
+; GFX1010-SDAG-NEXT: v_writelane_b32 v0, 0, s2
; GFX1010-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX1010-SDAG-NEXT: s_endpgm
;
; GFX1100-SDAG-LABEL: test_writelane_imm_sreg_f64:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_clause 0x1
-; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX1100-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x8
-; GFX1100-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1100-SDAG-NEXT: s_mov_b32 s3, 0x40400000
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX1100-SDAG-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, s3
-; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, s2
-; GFX1100-SDAG-NEXT: s_mov_b32 s2, 0x40400000
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1100-SDAG-NEXT: v_writelane_b32 v1, s2, s4
-; GFX1100-SDAG-NEXT: v_writelane_b32 v0, 0, s4
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, s5
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX1100-SDAG-NEXT: v_writelane_b32 v1, s3, s2
+; GFX1100-SDAG-NEXT: v_writelane_b32 v0, 0, s2
; GFX1100-SDAG-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1100-SDAG-NEXT: s_endpgm
;
; GFX802-GISEL-LABEL: test_writelane_imm_sreg_f64:
; GFX802-GISEL: ; %bb.0:
-; GFX802-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX802-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
+; GFX802-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX802-GISEL-NEXT: s_add_i32 s12, s12, s17
-; GFX802-GISEL-NEXT: s_mov_b32 s5, 0x40400000
-; GFX802-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX802-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
-; GFX802-GISEL-NEXT: s_mov_b32 m0, s4
-; GFX802-GISEL-NEXT: v_mov_b32_e32 v3, s1
+; GFX802-GISEL-NEXT: s_mov_b32 s3, 0x40400000
+; GFX802-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX802-GISEL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
+; GFX802-GISEL-NEXT: s_mov_b32 m0, s2
+; GFX802-GISEL-NEXT: v_mov_b32_e32 v3, s1
; GFX802-GISEL-NEXT: v_mov_b32_e32 v2, s0
; GFX802-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX802-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX802-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX802-GISEL-NEXT: v_writelane_b32 v0, 0, s4
-; GFX802-GISEL-NEXT: v_writelane_b32 v1, s5, m0
+; GFX802-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX802-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX802-GISEL-NEXT: v_writelane_b32 v0, 0, s2
+; GFX802-GISEL-NEXT: v_writelane_b32 v1, s3, m0
; GFX802-GISEL-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; GFX802-GISEL-NEXT: s_endpgm
;
; GFX1010-GISEL-LABEL: test_writelane_imm_sreg_f64:
; GFX1010-GISEL: ; %bb.0:
-; GFX1010-GISEL-NEXT: s_clause 0x1
-; GFX1010-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX1010-GISEL-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX1010-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1010-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX1010-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1010-GISEL-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x0
+; GFX1010-GISEL-NEXT: s_mov_b32 s3, 0x40400000
+; GFX1010-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1010-GISEL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x0
; GFX1010-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1010-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX1010-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX1010-GISEL-NEXT: s_mov_b32 s2, 0x40400000
-; GFX1010-GISEL-NEXT: v_writelane_b32 v0, 0, s4
-; GFX1010-GISEL-NEXT: v_writelane_b32 v1, s2, s4
+; GFX1010-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX1010-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX1010-GISEL-NEXT: v_writelane_b32 v0, 0, s2
+; GFX1010-GISEL-NEXT: v_writelane_b32 v1, s3, s2
; GFX1010-GISEL-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX1010-GISEL-NEXT: s_endpgm
;
; GFX1100-GISEL-LABEL: test_writelane_imm_sreg_f64:
; GFX1100-GISEL: ; %bb.0:
-; GFX1100-GISEL-NEXT: s_clause 0x1
-; GFX1100-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX1100-GISEL-NEXT: s_load_b32 s4, s[4:5], 0x8
-; GFX1100-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1100-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX1100-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-GISEL-NEXT: s_load_b64 s[2:3], s[0:1], 0x0
+; GFX1100-GISEL-NEXT: s_mov_b32 s3, 0x40400000
+; GFX1100-GISEL-NEXT: v_mov_b32_e32 v2, 0
+; GFX1100-GISEL-NEXT: s_load_b64 s[4:5], s[0:1], 0x0
; GFX1100-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-GISEL-NEXT: v_mov_b32_e32 v0, s2
-; GFX1100-GISEL-NEXT: v_mov_b32_e32 v1, s3
-; GFX1100-GISEL-NEXT: s_mov_b32 s2, 0x40400000
-; GFX1100-GISEL-NEXT: v_writelane_b32 v0, 0, s4
-; GFX1100-GISEL-NEXT: v_writelane_b32 v1, s2, s4
+; GFX1100-GISEL-NEXT: v_mov_b32_e32 v0, s4
+; GFX1100-GISEL-NEXT: v_mov_b32_e32 v1, s5
+; GFX1100-GISEL-NEXT: v_writelane_b32 v0, 0, s2
+; GFX1100-GISEL-NEXT: v_writelane_b32 v1, s3, s2
; GFX1100-GISEL-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1100-GISEL-NEXT: s_endpgm
%oldval = load double, ptr addrspace(1) %out
@@ -1076,31 +1045,28 @@ define amdgpu_kernel void @test_writelane_vreg_lane_f64(ptr addrspace(1) %out, p
define amdgpu_kernel void @test_writelane_m0_sreg_i32(ptr addrspace(1) %out, i32 %src1) #1 {
; GFX802-SDAG-LABEL: test_writelane_m0_sreg_i32:
; GFX802-SDAG: ; %bb.0:
-; GFX802-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX802-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX802-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX802-SDAG-NEXT: ;;#ASMSTART
; GFX802-SDAG-NEXT: s_mov_b32 m0, -1
; GFX802-SDAG-NEXT: ;;#ASMEND
; GFX802-SDAG-NEXT: s_add_i32 s12, s12, s17
; GFX802-SDAG-NEXT: s_mov_b32 s4, m0
+; GFX802-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX802-SDAG-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX802-SDAG-NEXT: s_mov_b32 m0, s2
; GFX802-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX802-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX802-SDAG-NEXT: v_mov_b32_e32 v1, s1
; GFX802-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX802-SDAG-NEXT: v_mov_b32_e32 v2, s3
; GFX802-SDAG-NEXT: v_writelane_b32 v2, s4, m0
-; GFX802-SDAG-NEXT: v_mov_b32_e32 v1, s1
; GFX802-SDAG-NEXT: flat_store_dword v[0:1], v2
; GFX802-SDAG-NEXT: s_endpgm
;
; GFX1010-SDAG-LABEL: test_writelane_m0_sreg_i32:
; GFX1010-SDAG: ; %bb.0:
-; GFX1010-SDAG-NEXT: s_clause 0x1
-; GFX1010-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX1010-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX1010-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX1010-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1010-SDAG-NEXT: ;;#ASMSTART
; GFX1010-SDAG-NEXT: s_mov_b32 m0, -1
@@ -1115,9 +1081,7 @@ define amdgpu_kernel void @test_writelane_m0_sreg_i32(ptr addrspace(1) %out, i32
;
; GFX1100-SDAG-LABEL: test_writelane_m0_sreg_i32:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_clause 0x1
-; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX1100-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1100-SDAG-NEXT: ;;#ASMSTART
; GFX1100-SDAG-NEXT: s_mov_b32 m0, -1
@@ -1132,31 +1096,28 @@ define amdgpu_kernel void @test_writelane_m0_sreg_i32(ptr addrspace(1) %out, i32
;
; GFX802-GISEL-LABEL: test_writelane_m0_sreg_i32:
; GFX802-GISEL: ; %bb.0:
-; GFX802-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX802-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX802-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX802-GISEL-NEXT: ;;#ASMSTART
; GFX802-GISEL-NEXT: s_mov_b32 m0, -1
; GFX802-GISEL-NEXT: ;;#ASMEND
; GFX802-GISEL-NEXT: s_add_i32 s12, s12, s17
; GFX802-GISEL-NEXT: s_mov_b32 s4, m0
+; GFX802-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX802-GISEL-NEXT: s_load_dword s3, s[0:1], 0x0
; GFX802-GISEL-NEXT: s_mov_b32 m0, s2
; GFX802-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; GFX802-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GFX802-GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX802-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX802-GISEL-NEXT: v_mov_b32_e32 v2, s3
; GFX802-GISEL-NEXT: v_writelane_b32 v2, s4, m0
-; GFX802-GISEL-NEXT: v_mov_b32_e32 v1, s1
; GFX802-GISEL-NEXT: flat_store_dword v[0:1], v2
; GFX802-GISEL-NEXT: s_endpgm
;
; GFX1010-GISEL-LABEL: test_writelane_m0_sreg_i32:
; GFX1010-GISEL: ; %bb.0:
-; GFX1010-GISEL-NEXT: s_clause 0x1
-; GFX1010-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX1010-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX1010-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX1010-GISEL-NEXT: ;;#ASMSTART
; GFX1010-GISEL-NEXT: s_mov_b32 m0, -1
; GFX1010-GISEL-NEXT: ;;#ASMEND
@@ -1171,9 +1132,7 @@ define amdgpu_kernel void @test_writelane_m0_sreg_i32(ptr addrspace(1) %out, i32
;
; GFX1100-GISEL-LABEL: test_writelane_m0_sreg_i32:
; GFX1100-GISEL: ; %bb.0:
-; GFX1100-GISEL-NEXT: s_clause 0x1
-; GFX1100-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX1100-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX1100-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX1100-GISEL-NEXT: ;;#ASMSTART
; GFX1100-GISEL-NEXT: s_mov_b32 m0, -1
; GFX1100-GISEL-NEXT: ;;#ASMEND
@@ -1195,8 +1154,7 @@ define amdgpu_kernel void @test_writelane_m0_sreg_i32(ptr addrspace(1) %out, i32
define amdgpu_kernel void @test_writelane_imm_i32(ptr addrspace(1) %out, i32 %src0) #1 {
; GFX802-SDAG-LABEL: test_writelane_imm_i32:
; GFX802-SDAG: ; %bb.0:
-; GFX802-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX802-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX802-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX802-SDAG-NEXT: s_add_i32 s12, s12, s17
; GFX802-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -1212,9 +1170,7 @@ define amdgpu_kernel void @test_writelane_imm_i32(ptr addrspace(1) %out, i32 %sr
;
; GFX1010-SDAG-LABEL: test_writelane_imm_i32:
; GFX1010-SDAG: ; %bb.0:
-; GFX1010-SDAG-NEXT: s_clause 0x1
-; GFX1010-SDAG-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX1010-SDAG-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX1010-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX1010-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1010-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1010-SDAG-NEXT: s_load_dword s3, s[0:1], 0x0
@@ -1226,9 +1182,7 @@ define amdgpu_kernel void @test_writelane_imm_i32(ptr addrspace(1) %out, i32 %sr
;
; GFX1100-SDAG-LABEL: test_writelane_imm_i32:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_clause 0x1
-; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX1100-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, 0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: s_load_b32 s3, s[0:1], 0x0
@@ -1240,8 +1194,7 @@ define amdgpu_kernel void @test_writelane_imm_i32(ptr addrspace(1) %out, i32 %sr
;
; GFX802-GISEL-LABEL: test_writelane_imm_i32:
; GFX802-GISEL: ; %bb.0:
-; GFX802-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX802-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX802-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX802-GISEL-NEXT: s_add_i32 s12, s12, s17
; GFX802-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
@@ -1257,9 +1210,7 @@ define amdgpu_kernel void @test_writelane_imm_i32(ptr addrspace(1) %out, i32 %sr
;
; GFX1010-GISEL-LABEL: test_writelane_imm_i32:
; GFX1010-GISEL: ; %bb.0:
-; GFX1010-GISEL-NEXT: s_clause 0x1
-; GFX1010-GISEL-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GFX1010-GISEL-NEXT: s_load_dword s2, s[8:9], 0x8
+; GFX1010-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX1010-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1010-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1010-GISEL-NEXT: s_load_dword s3, s[0:1], 0x0
@@ -1271,9 +1222,7 @@ define amdgpu_kernel void @test_writelane_imm_i32(ptr addrspace(1) %out, i32 %sr
;
; GFX1100-GISEL-LABEL: test_writelane_imm_i32:
; GFX1100-GISEL: ; %bb.0:
-; GFX1100-GISEL-NEXT: s_clause 0x1
-; GFX1100-GISEL-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX1100-GISEL-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX1100-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX1100-GISEL-NEXT: v_mov_b32_e32 v1, 0
; GFX1100-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-GISEL-NEXT: s_load_b32 s3, s[0:1], 0x0
@@ -1487,17 +1436,17 @@ define amdgpu_kernel void @test_writelane_imm_f64(ptr addrspace(1) %out, double
define amdgpu_kernel void @test_writelane_sreg_oldval_i32(i32 %oldval, ptr addrspace(1) %out, i32 %src0, i32 %src1) #1 {
; GFX802-SDAG-LABEL: test_writelane_sreg_oldval_i32:
; GFX802-SDAG: ; %bb.0:
-; GFX802-SDAG-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX802-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
+; GFX802-SDAG-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX802-SDAG-NEXT: s_add_i32 s12, s12, s17
; GFX802-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
; GFX802-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX802-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX802-SDAG-NEXT: v_mov_b32_e32 v2, s4
; GFX802-SDAG-NEXT: s_mov_b32 m0, s3
+; GFX802-SDAG-NEXT: v_mov_b32_e32 v2, s4
; GFX802-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX802-SDAG-NEXT: v_writelane_b32 v2, s2, m0
; GFX802-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX802-SDAG-NEXT: v_writelane_b32 v2, s2, m0
; GFX802-SDAG-NEXT: flat_store_dword v[0:1], v2
; GFX802-SDAG-NEXT: s_endpgm
;
@@ -1572,16 +1521,14 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_i32(i32 %oldval, ptr addrs
define amdgpu_kernel void @test_writelane_sreg_oldval_i64(i64 %oldval, ptr addrspace(1) %out, i64 %src0, i32 %src1) #1 {
; GFX802-SDAG-LABEL: test_writelane_sreg_oldval_i64:
; GFX802-SDAG: ; %bb.0:
-; GFX802-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX802-SDAG-NEXT: s_load_dword s6, s[8:9], 0x18
-; GFX802-SDAG-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX802-SDAG-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX802-SDAG-NEXT: s_add_i32 s12, s12, s17
; GFX802-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX802-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX802-SDAG-NEXT: v_mov_b32_e32 v3, s1
; GFX802-SDAG-NEXT: s_mov_b32 m0, s6
+; GFX802-SDAG-NEXT: v_mov_b32_e32 v3, s1
; GFX802-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX802-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX802-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX802-SDAG-NEXT: v_mov_b32_e32 v1, s3
; GFX802-SDAG-NEXT: v_writelane_b32 v3, s5, m0
@@ -1591,10 +1538,7 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_i64(i64 %oldval, ptr addrs
;
; GFX1010-SDAG-LABEL: test_writelane_sreg_oldval_i64:
; GFX1010-SDAG: ; %bb.0:
-; GFX1010-SDAG-NEXT: s_clause 0x2
-; GFX1010-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX1010-SDAG-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX1010-SDAG-NEXT: s_load_dword s6, s[8:9], 0x18
+; GFX1010-SDAG-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX1010-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1010-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1010-SDAG-NEXT: v_mov_b32_e32 v1, s1
@@ -1606,32 +1550,27 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_i64(i64 %oldval, ptr addrs
;
; GFX1100-SDAG-LABEL: test_writelane_sreg_oldval_i64:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_clause 0x2
-; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1100-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
-; GFX1100-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x18
+; GFX1100-SDAG-NEXT: s_load_b256 s[0:7], s[4:5], 0x0
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, s1
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX1100-SDAG-NEXT: v_writelane_b32 v1, s7, s4
-; GFX1100-SDAG-NEXT: v_writelane_b32 v0, s6, s4
+; GFX1100-SDAG-NEXT: v_writelane_b32 v1, s5, s6
+; GFX1100-SDAG-NEXT: v_writelane_b32 v0, s4, s6
; GFX1100-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX1100-SDAG-NEXT: s_endpgm
;
; GFX802-GISEL-LABEL: test_writelane_sreg_oldval_i64:
; GFX802-GISEL: ; %bb.0:
-; GFX802-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX802-GISEL-NEXT: s_load_dword s6, s[8:9], 0x18
-; GFX802-GISEL-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX802-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX802-GISEL-NEXT: s_add_i32 s12, s12, s17
; GFX802-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX802-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX802-GISEL-NEXT: s_mov_b32 m0, s6
; GFX802-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX802-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX802-GISEL-NEXT: s_mov_b32 m0, s6
; GFX802-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX802-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX802-GISEL-NEXT: v_writelane_b32 v0, s4, m0
; GFX802-GISEL-NEXT: v_writelane_b32 v1, s5, m0
; GFX802-GISEL-NEXT: v_mov_b32_e32 v3, s3
@@ -1640,10 +1579,7 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_i64(i64 %oldval, ptr addrs
;
; GFX1010-GISEL-LABEL: test_writelane_sreg_oldval_i64:
; GFX1010-GISEL: ; %bb.0:
-; GFX1010-GISEL-NEXT: s_clause 0x2
-; GFX1010-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX1010-GISEL-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX1010-GISEL-NEXT: s_load_dword s6, s[8:9], 0x18
+; GFX1010-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX1010-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1010-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1010-GISEL-NEXT: v_mov_b32_e32 v0, s0
@@ -1655,16 +1591,13 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_i64(i64 %oldval, ptr addrs
;
; GFX1100-GISEL-LABEL: test_writelane_sreg_oldval_i64:
; GFX1100-GISEL: ; %bb.0:
-; GFX1100-GISEL-NEXT: s_clause 0x2
-; GFX1100-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1100-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
-; GFX1100-GISEL-NEXT: s_load_b32 s4, s[4:5], 0x18
+; GFX1100-GISEL-NEXT: s_load_b256 s[0:7], s[4:5], 0x0
; GFX1100-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1100-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX1100-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1100-GISEL-NEXT: v_writelane_b32 v0, s6, s4
-; GFX1100-GISEL-NEXT: v_writelane_b32 v1, s7, s4
+; GFX1100-GISEL-NEXT: v_writelane_b32 v0, s4, s6
+; GFX1100-GISEL-NEXT: v_writelane_b32 v1, s5, s6
; GFX1100-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX1100-GISEL-NEXT: s_endpgm
%writelane = call i64 @llvm.amdgcn.writelane.i64(i64 %src0, i32 %src1, i64 %oldval)
@@ -1675,16 +1608,14 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_i64(i64 %oldval, ptr addrs
define amdgpu_kernel void @test_writelane_sreg_oldval_f64(double %oldval, ptr addrspace(1) %out, double %src0, i32 %src1) #1 {
; GFX802-SDAG-LABEL: test_writelane_sreg_oldval_f64:
; GFX802-SDAG: ; %bb.0:
-; GFX802-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX802-SDAG-NEXT: s_load_dword s6, s[8:9], 0x18
-; GFX802-SDAG-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX802-SDAG-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX802-SDAG-NEXT: s_add_i32 s12, s12, s17
; GFX802-SDAG-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GFX802-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX802-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX802-SDAG-NEXT: v_mov_b32_e32 v3, s1
; GFX802-SDAG-NEXT: s_mov_b32 m0, s6
+; GFX802-SDAG-NEXT: v_mov_b32_e32 v3, s1
; GFX802-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; GFX802-SDAG-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX802-SDAG-NEXT: v_mov_b32_e32 v0, s2
; GFX802-SDAG-NEXT: v_mov_b32_e32 v1, s3
; GFX802-SDAG-NEXT: v_writelane_b32 v3, s5, m0
@@ -1694,10 +1625,7 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_f64(double %oldval, ptr ad
;
; GFX1010-SDAG-LABEL: test_writelane_sreg_oldval_f64:
; GFX1010-SDAG: ; %bb.0:
-; GFX1010-SDAG-NEXT: s_clause 0x2
-; GFX1010-SDAG-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX1010-SDAG-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX1010-SDAG-NEXT: s_load_dword s6, s[8:9], 0x18
+; GFX1010-SDAG-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX1010-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1010-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1010-SDAG-NEXT: v_mov_b32_e32 v1, s1
@@ -1709,32 +1637,27 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_f64(double %oldval, ptr ad
;
; GFX1100-SDAG-LABEL: test_writelane_sreg_oldval_f64:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_clause 0x2
-; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1100-SDAG-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
-; GFX1100-SDAG-NEXT: s_load_b32 s4, s[4:5], 0x18
+; GFX1100-SDAG-NEXT: s_load_b256 s[0:7], s[4:5], 0x0
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v1, s1
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX1100-SDAG-NEXT: v_writelane_b32 v1, s7, s4
-; GFX1100-SDAG-NEXT: v_writelane_b32 v0, s6, s4
+; GFX1100-SDAG-NEXT: v_writelane_b32 v1, s5, s6
+; GFX1100-SDAG-NEXT: v_writelane_b32 v0, s4, s6
; GFX1100-SDAG-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX1100-SDAG-NEXT: s_endpgm
;
; GFX802-GISEL-LABEL: test_writelane_sreg_oldval_f64:
; GFX802-GISEL: ; %bb.0:
-; GFX802-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX802-GISEL-NEXT: s_load_dword s6, s[8:9], 0x18
-; GFX802-GISEL-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX802-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX802-GISEL-NEXT: s_add_i32 s12, s12, s17
; GFX802-GISEL-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GFX802-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX802-GISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX802-GISEL-NEXT: s_mov_b32 m0, s6
; GFX802-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX802-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX802-GISEL-NEXT: s_mov_b32 m0, s6
; GFX802-GISEL-NEXT: v_mov_b32_e32 v2, s2
-; GFX802-GISEL-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GFX802-GISEL-NEXT: v_writelane_b32 v0, s4, m0
; GFX802-GISEL-NEXT: v_writelane_b32 v1, s5, m0
; GFX802-GISEL-NEXT: v_mov_b32_e32 v3, s3
@@ -1743,10 +1666,7 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_f64(double %oldval, ptr ad
;
; GFX1010-GISEL-LABEL: test_writelane_sreg_oldval_f64:
; GFX1010-GISEL: ; %bb.0:
-; GFX1010-GISEL-NEXT: s_clause 0x2
-; GFX1010-GISEL-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
-; GFX1010-GISEL-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
-; GFX1010-GISEL-NEXT: s_load_dword s6, s[8:9], 0x18
+; GFX1010-GISEL-NEXT: s_load_dwordx8 s[0:7], s[8:9], 0x0
; GFX1010-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1010-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1010-GISEL-NEXT: v_mov_b32_e32 v0, s0
@@ -1758,16 +1678,13 @@ define amdgpu_kernel void @test_writelane_sreg_oldval_f64(double %oldval, ptr ad
;
; GFX1100-GISEL-LABEL: test_writelane_sreg_oldval_f64:
; GFX1100-GISEL: ; %bb.0:
-; GFX1100-GISEL-NEXT: s_clause 0x2
-; GFX1100-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
-; GFX1100-GISEL-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
-; GFX1100-GISEL-NEXT: s_load_b32 s4, s[4:5], 0x18
+; GFX1100-GISEL-NEXT: s_load_b256 s[0:7], s[4:5], 0x0
; GFX1100-GISEL-NEXT: v_mov_b32_e32 v2, 0
; GFX1100-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-GISEL-NEXT: v_mov_b32_e32 v0, s0
; GFX1100-GISEL-NEXT: v_mov_b32_e32 v1, s1
-; GFX1100-GISEL-NEXT: v_writelane_b32 v0, s6, s4
-; GFX1100-GISEL-NEXT: v_writelane_b32 v1, s7, s4
+; GFX1100-GISEL-NEXT: v_writelane_b32 v0, s4, s6
+; GFX1100-GISEL-NEXT: v_writelane_b32 v1, s5, s6
; GFX1100-GISEL-NEXT: global_store_b64 v2, v[0:1], s[2:3]
; GFX1100-GISEL-NEXT: s_endpgm
%writelane = call double @llvm.amdgcn.writelane.f64(double %src0, i32 %src1, double %oldval)
@@ -1786,8 +1703,8 @@ define amdgpu_kernel void @test_writelane_imm_oldval_i32(ptr addrspace(1) %out,
; GFX802-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX802-SDAG-NEXT: s_mov_b32 m0, s3
; GFX802-SDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX802-SDAG-NEXT: v_writelane_b32 v2, s2, m0
; GFX802-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX802-SDAG-NEXT: v_writelane_b32 v2, s2, m0
; GFX802-SDAG-NEXT: flat_store_dword v[0:1], v2
; GFX802-SDAG-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
index 91e4c0f1d7f7a..a02421dd50e3f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp.ll
@@ -12,36 +12,34 @@
define amdgpu_kernel void @s_exp_f32(ptr addrspace(1) %out, float %in) {
; VI-SDAG-LABEL: s_exp_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x3fb8a000
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7f800000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: s_and_b32 s0, s2, 0xfffff000
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s0
+; VI-SDAG-NEXT: s_and_b32 s3, s2, 0xfffff000
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
; VI-SDAG-NEXT: v_sub_f32_e32 v1, s2, v1
; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x39a3b295, v1
; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8a000, v1
-; VI-SDAG-NEXT: v_mul_f32_e32 v0, s0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, s3, v0
; VI-SDAG-NEXT: v_add_f32_e32 v1, v1, v3
; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x39a3b295
; VI-SDAG-NEXT: v_rndne_f32_e32 v2, v0
-; VI-SDAG-NEXT: v_mul_f32_e32 v3, s0, v3
+; VI-SDAG-NEXT: v_mul_f32_e32 v3, s3, v3
; VI-SDAG-NEXT: v_sub_f32_e32 v0, v0, v2
; VI-SDAG-NEXT: v_add_f32_e32 v1, v3, v1
; VI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
-; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
-; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v1, v2
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; VI-SDAG-NEXT: v_ldexp_f32 v0, v0, v1
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0xc2ce8ed0
-; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v1
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42b17218
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v1
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_exp_f32_e32 v3, v0
+; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v2
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_ldexp_f32 v2, v3, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0xc2ce8ed0
+; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v3
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x42b17218
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v3
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
; VI-SDAG-NEXT: s_endpgm
;
@@ -82,8 +80,7 @@ define amdgpu_kernel void @s_exp_f32(ptr addrspace(1) %out, float %in) {
;
; GFX900-SDAG-LABEL: s_exp_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x3fb8aa3b
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x32a5705f
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -95,14 +92,14 @@ define amdgpu_kernel void @s_exp_f32(ptr addrspace(1) %out, float %in) {
; GFX900-SDAG-NEXT: v_add_f32_e32 v0, v2, v0
; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v1, v3
; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, 0xc2ce8ed0
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v3
; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, 0x7f800000
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0xc2ce8ed0
-; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v1
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x42b17218
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v1
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
; GFX900-SDAG-NEXT: global_store_dword v2, v0, s[0:1]
; GFX900-SDAG-NEXT: s_endpgm
@@ -136,30 +133,31 @@ define amdgpu_kernel void @s_exp_f32(ptr addrspace(1) %out, float %in) {
;
; SI-SDAG-LABEL: s_exp_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x3fb8aa3b
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x32a5705f
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_mul_f32_e32 v2, s6, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, s2, v0
; SI-SDAG-NEXT: v_rndne_f32_e32 v3, v2
-; SI-SDAG-NEXT: v_fma_f32 v0, s6, v0, -v2
+; SI-SDAG-NEXT: v_fma_f32 v0, s2, v0, -v2
; SI-SDAG-NEXT: v_sub_f32_e32 v2, v2, v3
-; SI-SDAG-NEXT: v_fma_f32 v0, s6, v1, v0
+; SI-SDAG-NEXT: v_fma_f32 v0, s2, v1, v0
; SI-SDAG-NEXT: v_add_f32_e32 v0, v2, v0
; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v1, v3
; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
; SI-SDAG-NEXT: v_ldexp_f32_e32 v0, v0, v1
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0xc2ce8ed0
-; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s6, v1
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v1
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42b17218
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s6, v1
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v1
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: s_exp_f32:
@@ -339,53 +337,53 @@ define amdgpu_kernel void @s_exp_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; VI-SDAG-LABEL: s_exp_v2f32:
; VI-SDAG: ; %bb.0:
; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x3fb8a000
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x3fb8a000
+; VI-SDAG-NEXT: v_mov_b32_e32 v6, 0x42b17218
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: s_and_b32 s4, s3, 0xfffff000
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s4
-; VI-SDAG-NEXT: v_sub_f32_e32 v2, s3, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x39a3b295, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3fb8a000, v2
-; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v4
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; VI-SDAG-NEXT: v_sub_f32_e32 v1, s3, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x39a3b295, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x3fb8a000, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, s4, v2
+; VI-SDAG-NEXT: v_add_f32_e32 v1, v1, v4
; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0x39a3b295
-; VI-SDAG-NEXT: v_mul_f32_e32 v1, s4, v0
+; VI-SDAG-NEXT: v_rndne_f32_e32 v3, v0
; VI-SDAG-NEXT: v_mul_f32_e32 v5, s4, v4
-; VI-SDAG-NEXT: s_and_b32 s4, s2, 0xfffff000
-; VI-SDAG-NEXT: v_rndne_f32_e32 v3, v1
-; VI-SDAG-NEXT: v_mov_b32_e32 v6, s4
-; VI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v2, v5, v2
-; VI-SDAG-NEXT: v_sub_f32_e32 v6, s2, v6
-; VI-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v0, s4, v0
-; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x39a3b295, v6
-; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3fb8a000, v6
-; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
-; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v3
-; VI-SDAG-NEXT: v_rndne_f32_e32 v5, v0
-; VI-SDAG-NEXT: v_add_f32_e32 v6, v6, v7
-; VI-SDAG-NEXT: v_mul_f32_e32 v4, s4, v4
-; VI-SDAG-NEXT: v_sub_f32_e32 v0, v0, v5
-; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v6
-; VI-SDAG-NEXT: v_add_f32_e32 v0, v0, v4
-; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
-; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v4, v5
-; VI-SDAG-NEXT: v_ldexp_f32 v1, v1, v2
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0xc2ce8ed0
-; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v2
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x42b17218
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v5, 0x7f800000
-; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v3
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
-; VI-SDAG-NEXT: v_ldexp_f32 v0, v0, v4
-; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v2
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v3
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; VI-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-SDAG-NEXT: v_sub_f32_e32 v0, v0, v3
+; VI-SDAG-NEXT: v_add_f32_e32 v1, v5, v1
+; VI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v5, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: s_and_b32 s0, s2, 0xfffff000
+; VI-SDAG-NEXT: v_mov_b32_e32 v8, s0
+; VI-SDAG-NEXT: v_sub_f32_e32 v8, s2, v8
+; VI-SDAG-NEXT: v_mul_f32_e32 v2, s0, v2
+; VI-SDAG-NEXT: v_mul_f32_e32 v9, 0x39a3b295, v8
+; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x3fb8a000, v8
+; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
+; VI-SDAG-NEXT: v_rndne_f32_e32 v7, v2
+; VI-SDAG-NEXT: v_add_f32_e32 v8, v8, v9
+; VI-SDAG-NEXT: v_mul_f32_e32 v4, s0, v4
+; VI-SDAG-NEXT: v_sub_f32_e32 v2, v2, v7
+; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v8
+; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v4
+; VI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v4, v7
+; VI-SDAG-NEXT: v_ldexp_f32 v3, v5, v3
+; VI-SDAG-NEXT: v_mov_b32_e32 v5, 0xc2ce8ed0
+; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; VI-SDAG-NEXT: v_mov_b32_e32 v7, 0x7f800000
+; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v6
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
+; VI-SDAG-NEXT: v_ldexp_f32 v2, v2, v4
+; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v6
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc
+; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
; VI-GISEL-LABEL: s_exp_v2f32:
@@ -520,41 +518,42 @@ define amdgpu_kernel void @s_exp_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
;
; SI-SDAG-LABEL: s_exp_v2f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x3fb8aa3b
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x32a5705f
+; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s2, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-SDAG-NEXT: v_mul_f32_e32 v2, s5, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, s7, v0
; SI-SDAG-NEXT: v_rndne_f32_e32 v3, v2
-; SI-SDAG-NEXT: v_fma_f32 v4, s5, v0, -v2
+; SI-SDAG-NEXT: v_fma_f32 v4, s7, v0, -v2
; SI-SDAG-NEXT: v_sub_f32_e32 v2, v2, v3
-; SI-SDAG-NEXT: v_fma_f32 v4, s5, v1, v4
+; SI-SDAG-NEXT: v_fma_f32 v4, s7, v1, v4
; SI-SDAG-NEXT: v_add_f32_e32 v2, v2, v4
-; SI-SDAG-NEXT: v_mul_f32_e32 v5, s4, v0
-; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_mul_f32_e32 v5, s6, v0
; SI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
; SI-SDAG-NEXT: v_rndne_f32_e32 v6, v5
-; SI-SDAG-NEXT: v_fma_f32 v0, s4, v0, -v5
+; SI-SDAG-NEXT: v_fma_f32 v0, s6, v0, -v5
; SI-SDAG-NEXT: v_sub_f32_e32 v7, v5, v6
-; SI-SDAG-NEXT: v_fma_f32 v0, s4, v1, v0
+; SI-SDAG-NEXT: v_fma_f32 v0, s6, v1, v0
; SI-SDAG-NEXT: v_add_f32_e32 v0, v7, v0
; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v5, v6
; SI-SDAG-NEXT: v_ldexp_f32_e32 v2, v2, v3
; SI-SDAG-NEXT: v_mov_b32_e32 v3, 0xc2ce8ed0
-; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s5, v3
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s7, v3
; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x42b17218
; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0x7f800000
-; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s5, v4
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s7, v4
; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
; SI-SDAG-NEXT: v_ldexp_f32_e32 v0, v0, v5
-; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v3
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s6, v3
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v4
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s6, v4
+; SI-SDAG-NEXT: s_mov_b32 s0, s4
+; SI-SDAG-NEXT: s_mov_b32 s1, s5
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
; SI-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-SDAG-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
index 34799aed60793..ae92b5407da6c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp10.ll
@@ -14,36 +14,34 @@
define amdgpu_kernel void @s_exp10_f32(ptr addrspace(1) %out, float %in) {
; VI-SDAG-LABEL: s_exp10_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x40549000
+; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0x7f800000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: s_and_b32 s0, s2, 0xfffff000
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s0
+; VI-SDAG-NEXT: s_and_b32 s3, s2, 0xfffff000
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
; VI-SDAG-NEXT: v_sub_f32_e32 v1, s2, v1
; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3a2784bc, v1
; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549000, v1
-; VI-SDAG-NEXT: v_mul_f32_e32 v0, s0, v0
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, s3, v0
; VI-SDAG-NEXT: v_add_f32_e32 v1, v1, v3
; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x3a2784bc
; VI-SDAG-NEXT: v_rndne_f32_e32 v2, v0
-; VI-SDAG-NEXT: v_mul_f32_e32 v3, s0, v3
+; VI-SDAG-NEXT: v_mul_f32_e32 v3, s3, v3
; VI-SDAG-NEXT: v_sub_f32_e32 v0, v0, v2
; VI-SDAG-NEXT: v_add_f32_e32 v1, v3, v1
; VI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
-; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
-; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v1, v2
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; VI-SDAG-NEXT: v_ldexp_f32 v0, v0, v1
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0xc23369f4
-; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v1
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x421a209b
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v1
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
-; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: v_exp_f32_e32 v3, v0
+; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v2
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_ldexp_f32 v2, v3, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0xc23369f4
+; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v3
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x421a209b
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v3
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, v4, v2, vcc
; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
; VI-SDAG-NEXT: s_endpgm
;
@@ -84,8 +82,7 @@ define amdgpu_kernel void @s_exp10_f32(ptr addrspace(1) %out, float %in) {
;
; GFX900-SDAG-LABEL: s_exp10_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x40549a78
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x33979a37
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -97,14 +94,14 @@ define amdgpu_kernel void @s_exp10_f32(ptr addrspace(1) %out, float %in) {
; GFX900-SDAG-NEXT: v_add_f32_e32 v0, v2, v0
; GFX900-SDAG-NEXT: v_cvt_i32_f32_e32 v1, v3
; GFX900-SDAG-NEXT: v_exp_f32_e32 v0, v0
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, 0xc23369f4
+; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v3
; GFX900-SDAG-NEXT: v_mov_b32_e32 v3, 0x7f800000
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: v_ldexp_f32 v0, v0, v1
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0xc23369f4
-; GFX900-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v1
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x421a209b
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; GFX900-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v1
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, v3, v0, vcc
; GFX900-SDAG-NEXT: global_store_dword v2, v0, s[0:1]
; GFX900-SDAG-NEXT: s_endpgm
@@ -138,30 +135,31 @@ define amdgpu_kernel void @s_exp10_f32(ptr addrspace(1) %out, float %in) {
;
; SI-SDAG-LABEL: s_exp10_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x40549a78
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x33979a37
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_mul_f32_e32 v2, s6, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, s2, v0
; SI-SDAG-NEXT: v_rndne_f32_e32 v3, v2
-; SI-SDAG-NEXT: v_fma_f32 v0, s6, v0, -v2
+; SI-SDAG-NEXT: v_fma_f32 v0, s2, v0, -v2
; SI-SDAG-NEXT: v_sub_f32_e32 v2, v2, v3
-; SI-SDAG-NEXT: v_fma_f32 v0, s6, v1, v0
+; SI-SDAG-NEXT: v_fma_f32 v0, s2, v1, v0
; SI-SDAG-NEXT: v_add_f32_e32 v0, v2, v0
; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v1, v3
; SI-SDAG-NEXT: v_mov_b32_e32 v2, 0x7f800000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
; SI-SDAG-NEXT: v_ldexp_f32_e32 v0, v0, v1
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0xc23369f4
-; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s6, v1
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v1
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x421a209b
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s6, v1
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v1
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v2, v0, vcc
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: s_exp10_f32:
@@ -341,53 +339,53 @@ define amdgpu_kernel void @s_exp10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; VI-SDAG-LABEL: s_exp10_v2f32:
; VI-SDAG: ; %bb.0:
; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x40549000
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x40549000
+; VI-SDAG-NEXT: v_mov_b32_e32 v6, 0x421a209b
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: s_and_b32 s4, s3, 0xfffff000
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s4
-; VI-SDAG-NEXT: v_sub_f32_e32 v2, s3, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3a2784bc, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x40549000, v2
-; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v4
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; VI-SDAG-NEXT: v_sub_f32_e32 v1, s3, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3a2784bc, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v1, 0x40549000, v1
+; VI-SDAG-NEXT: v_mul_f32_e32 v0, s4, v2
+; VI-SDAG-NEXT: v_add_f32_e32 v1, v1, v4
; VI-SDAG-NEXT: v_mov_b32_e32 v4, 0x3a2784bc
-; VI-SDAG-NEXT: v_mul_f32_e32 v1, s4, v0
+; VI-SDAG-NEXT: v_rndne_f32_e32 v3, v0
; VI-SDAG-NEXT: v_mul_f32_e32 v5, s4, v4
-; VI-SDAG-NEXT: s_and_b32 s4, s2, 0xfffff000
-; VI-SDAG-NEXT: v_rndne_f32_e32 v3, v1
-; VI-SDAG-NEXT: v_mov_b32_e32 v6, s4
-; VI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v2, v5, v2
-; VI-SDAG-NEXT: v_sub_f32_e32 v6, s2, v6
-; VI-SDAG-NEXT: v_add_f32_e32 v1, v1, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v0, s4, v0
-; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3a2784bc, v6
-; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x40549000, v6
-; VI-SDAG-NEXT: v_exp_f32_e32 v1, v1
-; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v2, v3
-; VI-SDAG-NEXT: v_rndne_f32_e32 v5, v0
-; VI-SDAG-NEXT: v_add_f32_e32 v6, v6, v7
-; VI-SDAG-NEXT: v_mul_f32_e32 v4, s4, v4
-; VI-SDAG-NEXT: v_sub_f32_e32 v0, v0, v5
-; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v6
-; VI-SDAG-NEXT: v_add_f32_e32 v0, v0, v4
-; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
-; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v4, v5
-; VI-SDAG-NEXT: v_ldexp_f32 v1, v1, v2
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0xc23369f4
-; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v2
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x421a209b
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v5, 0x7f800000
-; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v3
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc
-; VI-SDAG-NEXT: v_ldexp_f32 v0, v0, v4
-; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v2
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v3
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, v5, v0, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; VI-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-SDAG-NEXT: v_sub_f32_e32 v0, v0, v3
+; VI-SDAG-NEXT: v_add_f32_e32 v1, v5, v1
+; VI-SDAG-NEXT: v_add_f32_e32 v0, v0, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v5, v0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: s_and_b32 s0, s2, 0xfffff000
+; VI-SDAG-NEXT: v_mov_b32_e32 v8, s0
+; VI-SDAG-NEXT: v_sub_f32_e32 v8, s2, v8
+; VI-SDAG-NEXT: v_mul_f32_e32 v2, s0, v2
+; VI-SDAG-NEXT: v_mul_f32_e32 v9, 0x3a2784bc, v8
+; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x40549000, v8
+; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
+; VI-SDAG-NEXT: v_rndne_f32_e32 v7, v2
+; VI-SDAG-NEXT: v_add_f32_e32 v8, v8, v9
+; VI-SDAG-NEXT: v_mul_f32_e32 v4, s0, v4
+; VI-SDAG-NEXT: v_sub_f32_e32 v2, v2, v7
+; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v8
+; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v4
+; VI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; VI-SDAG-NEXT: v_cvt_i32_f32_e32 v4, v7
+; VI-SDAG-NEXT: v_ldexp_f32 v3, v5, v3
+; VI-SDAG-NEXT: v_mov_b32_e32 v5, 0xc23369f4
+; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s3, v5
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, 0, v3, vcc
+; VI-SDAG-NEXT: v_mov_b32_e32 v7, 0x7f800000
+; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s3, v6
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v7, v3, vcc
+; VI-SDAG-NEXT: v_ldexp_f32 v2, v2, v4
+; VI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s2, v5
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; VI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s2, v6
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, v7, v2, vcc
+; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
; VI-GISEL-LABEL: s_exp10_v2f32:
@@ -522,41 +520,42 @@ define amdgpu_kernel void @s_exp10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
;
; SI-SDAG-LABEL: s_exp10_v2f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x40549a78
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x33979a37
+; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s2, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-SDAG-NEXT: v_mul_f32_e32 v2, s5, v0
+; SI-SDAG-NEXT: v_mul_f32_e32 v2, s7, v0
; SI-SDAG-NEXT: v_rndne_f32_e32 v3, v2
-; SI-SDAG-NEXT: v_fma_f32 v4, s5, v0, -v2
+; SI-SDAG-NEXT: v_fma_f32 v4, s7, v0, -v2
; SI-SDAG-NEXT: v_sub_f32_e32 v2, v2, v3
-; SI-SDAG-NEXT: v_fma_f32 v4, s5, v1, v4
+; SI-SDAG-NEXT: v_fma_f32 v4, s7, v1, v4
; SI-SDAG-NEXT: v_add_f32_e32 v2, v2, v4
-; SI-SDAG-NEXT: v_mul_f32_e32 v5, s4, v0
-; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
+; SI-SDAG-NEXT: v_mul_f32_e32 v5, s6, v0
; SI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v3, v3
; SI-SDAG-NEXT: v_rndne_f32_e32 v6, v5
-; SI-SDAG-NEXT: v_fma_f32 v0, s4, v0, -v5
+; SI-SDAG-NEXT: v_fma_f32 v0, s6, v0, -v5
; SI-SDAG-NEXT: v_sub_f32_e32 v7, v5, v6
-; SI-SDAG-NEXT: v_fma_f32 v0, s4, v1, v0
+; SI-SDAG-NEXT: v_fma_f32 v0, s6, v1, v0
; SI-SDAG-NEXT: v_add_f32_e32 v0, v7, v0
; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
; SI-SDAG-NEXT: v_cvt_i32_f32_e32 v5, v6
; SI-SDAG-NEXT: v_ldexp_f32_e32 v2, v2, v3
; SI-SDAG-NEXT: v_mov_b32_e32 v3, 0xc23369f4
-; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s5, v3
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s7, v3
; SI-SDAG-NEXT: v_mov_b32_e32 v4, 0x421a209b
; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
; SI-SDAG-NEXT: v_mov_b32_e32 v6, 0x7f800000
-; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s5, v4
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s7, v4
; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, v6, v2, vcc
; SI-SDAG-NEXT: v_ldexp_f32_e32 v0, v0, v5
-; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s4, v3
+; SI-SDAG-NEXT: v_cmp_nlt_f32_e32 vcc, s6, v3
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s4, v4
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: v_cmp_ngt_f32_e32 vcc, s6, v4
+; SI-SDAG-NEXT: s_mov_b32 s0, s4
+; SI-SDAG-NEXT: s_mov_b32 s1, s5
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
; SI-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-SDAG-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
index 53b862e8bab2f..568cda02fd70f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.exp2.ll
@@ -12,21 +12,22 @@
define amdgpu_kernel void @s_exp2_f32(ptr addrspace(1) %out, float %in) {
; SI-SDAG-LABEL: s_exp2_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42800000
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
; SI-SDAG-NEXT: v_add_f32_e32 v0, s2, v0
; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
-; SI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-SDAG-NEXT: s_cselect_b32 s4, 0xffffffc0, 0
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
-; SI-SDAG-NEXT: v_ldexp_f32_e64 v0, v0, s4
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
+; SI-SDAG-NEXT: v_ldexp_f32_e64 v0, v0, s0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: s_exp2_f32:
@@ -50,20 +51,19 @@ define amdgpu_kernel void @s_exp2_f32(ptr addrspace(1) %out, float %in) {
;
; VI-SDAG-LABEL: s_exp2_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42800000
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0xc2fc0000
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x42800000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_add_f32_e32 v0, s2, v0
-; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
-; VI-SDAG-NEXT: v_ldexp_f32 v2, v0, s2
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; VI-SDAG-NEXT: v_add_f32_e32 v1, s2, v1
+; VI-SDAG-NEXT: v_exp_f32_e32 v2, v1
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
+; VI-SDAG-NEXT: v_ldexp_f32 v2, v2, s0
; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
; VI-SDAG-NEXT: s_endpgm
;
@@ -88,8 +88,7 @@ define amdgpu_kernel void @s_exp2_f32(ptr addrspace(1) %out, float %in) {
;
; GFX900-SDAG-LABEL: s_exp2_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x42800000
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -176,25 +175,26 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42800000
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s5, v0
+; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; SI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
-; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT: v_add_f32_e32 v2, s5, v2
-; SI-SDAG-NEXT: v_add_f32_e32 v0, s4, v0
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
+; SI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-SDAG-NEXT: v_add_f32_e32 v2, s3, v2
+; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
; SI-SDAG-NEXT: v_exp_f32_e32 v2, v2
+; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
+; SI-SDAG-NEXT: v_add_f32_e32 v0, s2, v0
; SI-SDAG-NEXT: v_exp_f32_e32 v0, v0
-; SI-SDAG-NEXT: s_cselect_b32 s6, 0xffffffc0, 0
-; SI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
-; SI-SDAG-NEXT: s_cselect_b32 s4, 0xffffffc0, 0
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
-; SI-SDAG-NEXT: v_ldexp_f32_e64 v1, v2, s6
-; SI-SDAG-NEXT: v_ldexp_f32_e64 v0, v0, s4
-; SI-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-SDAG-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
+; SI-SDAG-NEXT: v_ldexp_f32_e64 v1, v2, s0
+; SI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
+; SI-SDAG-NEXT: v_ldexp_f32_e64 v0, v0, s0
+; SI-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: s_exp2_v2f32:
@@ -224,26 +224,26 @@ define amdgpu_kernel void @s_exp2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; VI-SDAG-LABEL: s_exp2_v2f32:
; VI-SDAG: ; %bb.0:
; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0xc2fc0000
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42800000
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0xc2fc0000
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x42800000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
-; VI-SDAG-NEXT: v_add_f32_e32 v2, s3, v2
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v2
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v3, vcc
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-SDAG-NEXT: v_add_f32_e32 v4, s3, v4
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v2
+; VI-SDAG-NEXT: v_exp_f32_e32 v4, v4
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v3, vcc
+; VI-SDAG-NEXT: v_add_f32_e32 v2, s2, v2
; VI-SDAG-NEXT: v_exp_f32_e32 v2, v2
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_add_f32_e32 v0, s2, v0
-; VI-SDAG-NEXT: v_exp_f32_e32 v0, v0
-; VI-SDAG-NEXT: s_cselect_b32 s3, 0xffffffc0, 0
-; VI-SDAG-NEXT: v_ldexp_f32 v1, v2, s3
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s2, 0xffffffc0, 0
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
-; VI-SDAG-NEXT: v_ldexp_f32 v0, v0, s2
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; VI-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-SDAG-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
+; VI-SDAG-NEXT: v_ldexp_f32 v3, v4, s0
+; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s0, 0xffffffc0, 0
+; VI-SDAG-NEXT: v_ldexp_f32 v2, v2, s0
+; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
; VI-GISEL-LABEL: s_exp2_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
index 52e9474f46777..274cbc8ebab2a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.bf16.ll
@@ -16,36 +16,35 @@
define amdgpu_kernel void @sgpr_isnan_bf16(ptr addrspace(1) %out, bfloat %x) {
; GFX7CHECK-LABEL: sgpr_isnan_bf16:
; GFX7CHECK: ; %bb.0:
-; GFX7CHECK-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7CHECK-NEXT: s_mov_b32 s3, 0xf000
-; GFX7CHECK-NEXT: s_mov_b32 s2, -1
+; GFX7CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7CHECK-NEXT: s_mov_b32 s7, 0xf000
+; GFX7CHECK-NEXT: s_mov_b32 s6, -1
; GFX7CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7CHECK-NEXT: s_and_b32 s4, s6, 0x7fff
-; GFX7CHECK-NEXT: s_cmpk_gt_i32 s4, 0x7f80
-; GFX7CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; GFX7CHECK-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7CHECK-NEXT: s_mov_b32 s4, s0
+; GFX7CHECK-NEXT: s_and_b32 s0, s2, 0x7fff
+; GFX7CHECK-NEXT: s_cmpk_gt_i32 s0, 0x7f80
+; GFX7CHECK-NEXT: s_mov_b32 s5, s1
+; GFX7CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX7CHECK-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; GFX7CHECK-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7CHECK-NEXT: s_endpgm
;
; GFX8CHECK-LABEL: sgpr_isnan_bf16:
; GFX8CHECK: ; %bb.0:
-; GFX8CHECK-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8CHECK-NEXT: s_and_b32 s2, s2, 0x7fff
-; GFX8CHECK-NEXT: s_cmpk_gt_i32 s2, 0x7f80
-; GFX8CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
; GFX8CHECK-NEXT: v_mov_b32_e32 v0, s0
-; GFX8CHECK-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[2:3]
+; GFX8CHECK-NEXT: s_and_b32 s0, s2, 0x7fff
+; GFX8CHECK-NEXT: s_cmpk_gt_i32 s0, 0x7f80
; GFX8CHECK-NEXT: v_mov_b32_e32 v1, s1
+; GFX8CHECK-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX8CHECK-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[0:1]
; GFX8CHECK-NEXT: flat_store_dword v[0:1], v2
; GFX8CHECK-NEXT: s_endpgm
;
; GFX9CHECK-LABEL: sgpr_isnan_bf16:
; GFX9CHECK: ; %bb.0:
-; GFX9CHECK-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9CHECK-NEXT: v_mov_b32_e32 v0, 0
; GFX9CHECK-NEXT: s_waitcnt lgkmcnt(0)
; GFX9CHECK-NEXT: s_and_b32 s2, s2, 0x7fff
@@ -57,9 +56,7 @@ define amdgpu_kernel void @sgpr_isnan_bf16(ptr addrspace(1) %out, bfloat %x) {
;
; GFX10CHECK-LABEL: sgpr_isnan_bf16:
; GFX10CHECK: ; %bb.0:
-; GFX10CHECK-NEXT: s_clause 0x1
-; GFX10CHECK-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10CHECK-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10CHECK-NEXT: v_mov_b32_e32 v0, 0
; GFX10CHECK-NEXT: s_waitcnt lgkmcnt(0)
; GFX10CHECK-NEXT: s_and_b32 s2, s2, 0x7fff
@@ -71,9 +68,7 @@ define amdgpu_kernel void @sgpr_isnan_bf16(ptr addrspace(1) %out, bfloat %x) {
;
; GFX11CHECK-LABEL: sgpr_isnan_bf16:
; GFX11CHECK: ; %bb.0:
-; GFX11CHECK-NEXT: s_clause 0x1
-; GFX11CHECK-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11CHECK-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11CHECK-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11CHECK-NEXT: v_mov_b32_e32 v0, 0
; GFX11CHECK-NEXT: s_waitcnt lgkmcnt(0)
; GFX11CHECK-NEXT: s_and_b32 s2, s2, 0x7fff
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
index b3224dd565fe5..944358b4fb26e 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.f16.ll
@@ -18,16 +18,17 @@
define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
; GFX7SELDAG-LABEL: sgpr_isnan_f16:
; GFX7SELDAG: ; %bb.0:
-; GFX7SELDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7SELDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX7SELDAG-NEXT: s_mov_b32 s2, -1
+; GFX7SELDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7SELDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX7SELDAG-NEXT: s_mov_b32 s6, -1
; GFX7SELDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7SELDAG-NEXT: s_and_b32 s4, s6, 0x7fff
-; GFX7SELDAG-NEXT: s_cmpk_gt_i32 s4, 0x7c00
-; GFX7SELDAG-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; GFX7SELDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7SELDAG-NEXT: s_mov_b32 s4, s0
+; GFX7SELDAG-NEXT: s_and_b32 s0, s2, 0x7fff
+; GFX7SELDAG-NEXT: s_cmpk_gt_i32 s0, 0x7c00
+; GFX7SELDAG-NEXT: s_mov_b32 s5, s1
+; GFX7SELDAG-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; GFX7SELDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7SELDAG-NEXT: s_endpgm
;
; GFX7GLISEL-LABEL: sgpr_isnan_f16:
@@ -47,13 +48,12 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
;
; GFX8SELDAG-LABEL: sgpr_isnan_f16:
; GFX8SELDAG: ; %bb.0:
-; GFX8SELDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8SELDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8SELDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8SELDAG-NEXT: v_cmp_class_f16_e64 s[2:3], s2, 3
; GFX8SELDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[2:3]
; GFX8SELDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX8SELDAG-NEXT: v_cmp_class_f16_e64 s[0:1], s2, 3
+; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[0:1]
; GFX8SELDAG-NEXT: flat_store_dword v[0:1], v2
; GFX8SELDAG-NEXT: s_endpgm
;
@@ -73,8 +73,7 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
;
; GFX9SELDAG-LABEL: sgpr_isnan_f16:
; GFX9SELDAG: ; %bb.0:
-; GFX9SELDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9SELDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9SELDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9SELDAG-NEXT: v_cmp_class_f16_e64 s[2:3], s2, 3
@@ -97,9 +96,7 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
;
; GFX10SELDAG-LABEL: sgpr_isnan_f16:
; GFX10SELDAG: ; %bb.0:
-; GFX10SELDAG-NEXT: s_clause 0x1
-; GFX10SELDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10SELDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10SELDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX10SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10SELDAG-NEXT: v_cmp_class_f16_e64 s2, s2, 3
@@ -123,9 +120,7 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
;
; GFX11SELDAG-TRUE16-LABEL: sgpr_isnan_f16:
; GFX11SELDAG-TRUE16: ; %bb.0:
-; GFX11SELDAG-TRUE16-NEXT: s_clause 0x1
-; GFX11SELDAG-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11SELDAG-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11SELDAG-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11SELDAG-TRUE16-NEXT: v_dual_mov_b32 v0, 3 :: v_dual_mov_b32 v1, 0
; GFX11SELDAG-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11SELDAG-TRUE16-NEXT: v_cmp_class_f16_e32 vcc_lo, s2, v0.l
@@ -135,9 +130,7 @@ define amdgpu_kernel void @sgpr_isnan_f16(ptr addrspace(1) %out, half %x) {
;
; GFX11SELDAG-FAKE16-LABEL: sgpr_isnan_f16:
; GFX11SELDAG-FAKE16: ; %bb.0:
-; GFX11SELDAG-FAKE16-NEXT: s_clause 0x1
-; GFX11SELDAG-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11SELDAG-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11SELDAG-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11SELDAG-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11SELDAG-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11SELDAG-FAKE16-NEXT: v_cmp_class_f16_e64 s2, s2, 3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
index fd7424ce05af4..3e0e243e12dae 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.is.fpclass.ll
@@ -16,14 +16,15 @@
define amdgpu_kernel void @sgpr_isnan_f32(ptr addrspace(1) %out, float %x) {
; GFX7SELDAG-LABEL: sgpr_isnan_f32:
; GFX7SELDAG: ; %bb.0:
-; GFX7SELDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX7SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX7SELDAG-NEXT: s_mov_b32 s3, 0xf000
-; GFX7SELDAG-NEXT: s_mov_b32 s2, -1
+; GFX7SELDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX7SELDAG-NEXT: s_mov_b32 s7, 0xf000
+; GFX7SELDAG-NEXT: s_mov_b32 s6, -1
; GFX7SELDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7SELDAG-NEXT: v_cmp_class_f32_e64 s[4:5], s6, 3
-; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; GFX7SELDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7SELDAG-NEXT: s_mov_b32 s4, s0
+; GFX7SELDAG-NEXT: s_mov_b32 s5, s1
+; GFX7SELDAG-NEXT: v_cmp_class_f32_e64 s[0:1], s2, 3
+; GFX7SELDAG-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; GFX7SELDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7SELDAG-NEXT: s_endpgm
;
; GFX7GLISEL-LABEL: sgpr_isnan_f32:
@@ -42,13 +43,12 @@ define amdgpu_kernel void @sgpr_isnan_f32(ptr addrspace(1) %out, float %x) {
;
; GFX8SELDAG-LABEL: sgpr_isnan_f32:
; GFX8SELDAG: ; %bb.0:
-; GFX8SELDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8SELDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8SELDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8SELDAG-NEXT: v_cmp_class_f32_e64 s[2:3], s2, 3
; GFX8SELDAG-NEXT: v_mov_b32_e32 v0, s0
-; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[2:3]
; GFX8SELDAG-NEXT: v_mov_b32_e32 v1, s1
+; GFX8SELDAG-NEXT: v_cmp_class_f32_e64 s[0:1], s2, 3
+; GFX8SELDAG-NEXT: v_cndmask_b32_e64 v2, 0, -1, s[0:1]
; GFX8SELDAG-NEXT: flat_store_dword v[0:1], v2
; GFX8SELDAG-NEXT: s_endpgm
;
@@ -68,8 +68,7 @@ define amdgpu_kernel void @sgpr_isnan_f32(ptr addrspace(1) %out, float %x) {
;
; GFX9SELDAG-LABEL: sgpr_isnan_f32:
; GFX9SELDAG: ; %bb.0:
-; GFX9SELDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9SELDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9SELDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX9SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX9SELDAG-NEXT: v_cmp_class_f32_e64 s[2:3], s2, 3
@@ -92,9 +91,7 @@ define amdgpu_kernel void @sgpr_isnan_f32(ptr addrspace(1) %out, float %x) {
;
; GFX10SELDAG-LABEL: sgpr_isnan_f32:
; GFX10SELDAG: ; %bb.0:
-; GFX10SELDAG-NEXT: s_clause 0x1
-; GFX10SELDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10SELDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10SELDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10SELDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX10SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX10SELDAG-NEXT: v_cmp_class_f32_e64 s2, s2, 3
@@ -118,9 +115,7 @@ define amdgpu_kernel void @sgpr_isnan_f32(ptr addrspace(1) %out, float %x) {
;
; GFX11SELDAG-LABEL: sgpr_isnan_f32:
; GFX11SELDAG: ; %bb.0:
-; GFX11SELDAG-NEXT: s_clause 0x1
-; GFX11SELDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11SELDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11SELDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11SELDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX11SELDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX11SELDAG-NEXT: v_cmp_class_f32_e64 s2, s2, 3
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log.ll b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
index cef8aa0c8dd38..39c2b29fc94fa 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log.ll
@@ -16,31 +16,32 @@
define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
; SI-SDAG-LABEL: s_log_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x3f317217
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; SI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; SI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s6, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s2, v1
; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x3f317217
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317217, v1
-; SI-SDAG-NEXT: v_fma_f32 v3, v1, s4, -v2
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x3377d1cf
-; SI-SDAG-NEXT: v_fma_f32 v3, v1, s4, v3
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-SDAG-NEXT: v_fma_f32 v3, v1, s0, -v2
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x3377d1cf
+; SI-SDAG-NEXT: v_fma_f32 v3, v1, s0, v3
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x7f800000
; SI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
-; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s4
+; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s0
; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; SI-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: s_log_f32:
@@ -73,34 +74,32 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
;
; VI-SDAG-LABEL: s_log_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s0
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
-; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
-; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v1
-; VI-SDAG-NEXT: v_sub_f32_e32 v3, v1, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3805fdf4, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x3f317000, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3805fdf4, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v4, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317000, v2
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v5, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s2
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; VI-SDAG-NEXT: v_sub_f32_e32 v2, v1, v0
-; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; VI-SDAG-NEXT: v_ldexp_f32 v0, s2, v0
+; VI-SDAG-NEXT: v_log_f32_e32 v3, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: s_mov_b32 s0, 0x7f800000
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
+; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
+; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3805fdf4, v4
+; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3f317000, v5
+; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x3805fdf4, v5
+; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
+; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
+; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3f317000, v4
+; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s0
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_sub_f32_e32 v2, v3, v2
; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
; VI-SDAG-NEXT: s_endpgm
;
@@ -137,30 +136,29 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
;
; GFX900-SDAG-LABEL: s_log_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x800000
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x41b17218
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
-; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
-; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
+; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, s3
+; GFX900-SDAG-NEXT: v_ldexp_f32 v2, s2, v2
+; GFX900-SDAG-NEXT: v_log_f32_e32 v2, v2
; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x3f317217
; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3377d1cf
-; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317217, v1
-; GFX900-SDAG-NEXT: v_fma_f32 v4, v1, s2, -v3
-; GFX900-SDAG-NEXT: v_fma_f32 v4, v1, s3, v4
-; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317217, v2
+; GFX900-SDAG-NEXT: v_fma_f32 v4, v2, s2, -v3
+; GFX900-SDAG-NEXT: v_fma_f32 v4, v2, s3, v4
; GFX900-SDAG-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s2
-; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX900-SDAG-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v2|, s4
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v2, v1
+; GFX900-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
; GFX900-SDAG-NEXT: s_endpgm
;
; GFX900-GISEL-LABEL: s_log_f32:
@@ -192,15 +190,14 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
;
; GFX1100-SDAG-LABEL: s_log_f32:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s1, 0x800000, s0
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s3, 0x800000, s2
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x41b17218, s1
-; GFX1100-SDAG-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX1100-SDAG-NEXT: s_cselect_b32 s1, 32, 0
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s0, s1
-; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x41b17218, s3
+; GFX1100-SDAG-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s2, s3
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -213,7 +210,6 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v2 :: v_dual_mov_b32 v2, 0
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX1100-SDAG-NEXT: s_endpgm
;
@@ -319,31 +315,32 @@ define amdgpu_kernel void @s_log_f32(ptr addrspace(1) %out, float %in) {
define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in) {
; SI-SDAG-LABEL: s_log_contract_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x3f317217
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; SI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; SI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s6, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s2, v1
; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x3f317217
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317217, v1
-; SI-SDAG-NEXT: v_fma_f32 v3, v1, s4, -v2
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x3377d1cf
-; SI-SDAG-NEXT: v_fma_f32 v3, v1, s4, v3
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-SDAG-NEXT: v_fma_f32 v3, v1, s0, -v2
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x3377d1cf
+; SI-SDAG-NEXT: v_fma_f32 v3, v1, s0, v3
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x7f800000
; SI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
-; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s4
+; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s0
; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; SI-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: s_log_contract_f32:
@@ -376,34 +373,32 @@ define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in)
;
; VI-SDAG-LABEL: s_log_contract_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s0
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
-; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
-; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v1
-; VI-SDAG-NEXT: v_sub_f32_e32 v3, v1, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3805fdf4, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x3f317000, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3805fdf4, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v4, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317000, v2
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v5, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s2
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; VI-SDAG-NEXT: v_sub_f32_e32 v2, v1, v0
-; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; VI-SDAG-NEXT: v_ldexp_f32 v0, s2, v0
+; VI-SDAG-NEXT: v_log_f32_e32 v3, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: s_mov_b32 s0, 0x7f800000
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
+; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
+; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3805fdf4, v4
+; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3f317000, v5
+; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x3805fdf4, v5
+; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
+; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
+; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3f317000, v4
+; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s0
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_sub_f32_e32 v2, v3, v2
; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
; VI-SDAG-NEXT: s_endpgm
;
@@ -440,30 +435,29 @@ define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in)
;
; GFX900-SDAG-LABEL: s_log_contract_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x800000
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x41b17218
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
-; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
-; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
+; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, s3
+; GFX900-SDAG-NEXT: v_ldexp_f32 v2, s2, v2
+; GFX900-SDAG-NEXT: v_log_f32_e32 v2, v2
; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x3f317217
; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3377d1cf
-; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317217, v1
-; GFX900-SDAG-NEXT: v_fma_f32 v4, v1, s2, -v3
-; GFX900-SDAG-NEXT: v_fma_f32 v4, v1, s3, v4
-; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3f317217, v2
+; GFX900-SDAG-NEXT: v_fma_f32 v4, v2, s2, -v3
+; GFX900-SDAG-NEXT: v_fma_f32 v4, v2, s3, v4
; GFX900-SDAG-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s2
-; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX900-SDAG-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v2|, s4
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v2, v1
+; GFX900-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
; GFX900-SDAG-NEXT: s_endpgm
;
; GFX900-GISEL-LABEL: s_log_contract_f32:
@@ -495,15 +489,14 @@ define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in)
;
; GFX1100-SDAG-LABEL: s_log_contract_f32:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s1, 0x800000, s0
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s3, 0x800000, s2
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x41b17218, s1
-; GFX1100-SDAG-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX1100-SDAG-NEXT: s_cselect_b32 s1, 32, 0
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s0, s1
-; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x41b17218, s3
+; GFX1100-SDAG-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s2, s3
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -516,7 +509,6 @@ define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in)
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v2 :: v_dual_mov_b32 v2, 0
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX1100-SDAG-NEXT: s_endpgm
;
@@ -624,38 +616,39 @@ define amdgpu_kernel void @s_log_contract_f32(ptr addrspace(1) %out, float %in)
define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; SI-SDAG-LABEL: s_log_v2f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
-; SI-SDAG-NEXT: s_mov_b32 s8, 0x3f317217
+; SI-SDAG-NEXT: s_mov_b32 s8, 0x3377d1cf
; SI-SDAG-NEXT: s_mov_b32 s9, 0x7f800000
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s5, v0
-; SI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; SI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
-; SI-SDAG-NEXT: v_mov_b32_e32 v3, s2
-; SI-SDAG-NEXT: v_ldexp_f32_e32 v3, s5, v3
+; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s7, v0
+; SI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
+; SI-SDAG-NEXT: v_mov_b32_e32 v3, s0
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v3, s7, v3
; SI-SDAG-NEXT: v_log_f32_e32 v3, v3
; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
+; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
+; SI-SDAG-NEXT: s_mov_b32 s0, s4
+; SI-SDAG-NEXT: s_mov_b32 s1, s5
+; SI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x3f317217
; SI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3f317217, v3
-; SI-SDAG-NEXT: s_cselect_b32 s6, 32, 0
-; SI-SDAG-NEXT: s_mov_b32 s5, 0x3377d1cf
-; SI-SDAG-NEXT: v_fma_f32 v5, v3, s8, -v4
+; SI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
+; SI-SDAG-NEXT: v_fma_f32 v5, v3, s7, -v4
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT: v_mov_b32_e32 v1, s6
-; SI-SDAG-NEXT: v_fma_f32 v5, v3, s5, v5
-; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s4, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; SI-SDAG-NEXT: v_fma_f32 v5, v3, s8, v5
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s6, v1
; SI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
; SI-SDAG-NEXT: v_log_f32_e32 v5, v1
; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s9
; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; SI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v2
; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317217, v5
-; SI-SDAG-NEXT: v_fma_f32 v3, v5, s8, -v2
-; SI-SDAG-NEXT: v_fma_f32 v3, v5, s5, v3
+; SI-SDAG-NEXT: v_fma_f32 v3, v5, s7, -v2
+; SI-SDAG-NEXT: v_fma_f32 v3, v5, s8, v3
; SI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v5|, s9
; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
@@ -708,51 +701,51 @@ define amdgpu_kernel void @s_log_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; VI-SDAG-LABEL: s_log_v2f32:
; VI-SDAG: ; %bb.0:
; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x41b17218
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x800000
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x41b17218
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v2
; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
; VI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s4
-; VI-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
-; VI-SDAG-NEXT: v_log_f32_e32 v3, v3
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
-; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
-; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
-; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3805fdf4, v4
-; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3f317000, v5
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: v_ldexp_f32 v0, s3, v0
+; VI-SDAG-NEXT: v_log_f32_e32 v5, v0
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v3, vcc
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: v_and_b32_e32 v6, 0xfffff000, v5
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_sub_f32_e32 v7, v5, v6
+; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x3805fdf4, v6
+; VI-SDAG-NEXT: v_mul_f32_e32 v9, 0x3f317000, v7
+; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3805fdf4, v7
+; VI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
+; VI-SDAG-NEXT: v_add_f32_e32 v7, v8, v7
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v3, vcc
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s0
+; VI-SDAG-NEXT: v_add_f32_e32 v7, v9, v7
+; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3f317000, v6
+; VI-SDAG-NEXT: v_ldexp_f32 v3, s2, v3
+; VI-SDAG-NEXT: v_add_f32_e32 v6, v6, v7
+; VI-SDAG-NEXT: v_log_f32_e32 v7, v3
+; VI-SDAG-NEXT: s_mov_b32 s3, 0x7f800000
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v5|, s3
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; VI-SDAG-NEXT: v_sub_f32_e32 v3, v3, v4
+; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v7
+; VI-SDAG-NEXT: v_sub_f32_e32 v5, v7, v4
+; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3f317000, v5
; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x3805fdf4, v5
-; VI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
+; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x3805fdf4, v4
+; VI-SDAG-NEXT: v_add_f32_e32 v5, v8, v5
; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s4
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3f317000, v4
-; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
-; VI-SDAG-NEXT: v_log_f32_e32 v5, v1
-; VI-SDAG-NEXT: s_mov_b32 s3, 0x7f800000
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s3
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; VI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v2
-; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v5
-; VI-SDAG-NEXT: v_sub_f32_e32 v3, v5, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3f317000, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x3805fdf4, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3805fdf4, v2
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v6, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v4, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3f317000, v2
-; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v5|, s3
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
-; VI-SDAG-NEXT: v_sub_f32_e32 v0, v2, v0
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; VI-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s3
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc
+; VI-SDAG-NEXT: v_sub_f32_e32 v2, v4, v2
+; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
; VI-GISEL-LABEL: s_log_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
index 7da9a6a9e4eea..fd72886743f20 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log10.ll
@@ -16,31 +16,32 @@
define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
; SI-SDAG-LABEL: s_log10_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x3e9a209a
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; SI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; SI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s6, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s2, v1
; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x3e9a209a
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209a, v1
-; SI-SDAG-NEXT: v_fma_f32 v3, v1, s4, -v2
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x3284fbcf
-; SI-SDAG-NEXT: v_fma_f32 v3, v1, s4, v3
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-SDAG-NEXT: v_fma_f32 v3, v1, s0, -v2
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x3284fbcf
+; SI-SDAG-NEXT: v_fma_f32 v3, v1, s0, v3
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x7f800000
; SI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
-; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s4
+; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s0
; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; SI-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: s_log10_f32:
@@ -73,34 +74,32 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
;
; VI-SDAG-LABEL: s_log10_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s0
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
-; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
-; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v1
-; VI-SDAG-NEXT: v_sub_f32_e32 v3, v1, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x369a84fb, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x3e9a2000, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x369a84fb, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v4, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a2000, v2
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v5, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s2
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; VI-SDAG-NEXT: v_sub_f32_e32 v2, v1, v0
-; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; VI-SDAG-NEXT: v_ldexp_f32 v0, s2, v0
+; VI-SDAG-NEXT: v_log_f32_e32 v3, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: s_mov_b32 s0, 0x7f800000
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
+; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
+; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x369a84fb, v4
+; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3e9a2000, v5
+; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x369a84fb, v5
+; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
+; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
+; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3e9a2000, v4
+; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s0
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_sub_f32_e32 v2, v3, v2
; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
; VI-SDAG-NEXT: s_endpgm
;
@@ -137,30 +136,29 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
;
; GFX900-SDAG-LABEL: s_log10_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x800000
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x411a209b
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
-; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
-; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
+; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, s3
+; GFX900-SDAG-NEXT: v_ldexp_f32 v2, s2, v2
+; GFX900-SDAG-NEXT: v_log_f32_e32 v2, v2
; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x3e9a209a
; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3284fbcf
-; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209a, v1
-; GFX900-SDAG-NEXT: v_fma_f32 v4, v1, s2, -v3
-; GFX900-SDAG-NEXT: v_fma_f32 v4, v1, s3, v4
-; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209a, v2
+; GFX900-SDAG-NEXT: v_fma_f32 v4, v2, s2, -v3
+; GFX900-SDAG-NEXT: v_fma_f32 v4, v2, s3, v4
; GFX900-SDAG-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s2
-; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX900-SDAG-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v2|, s4
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v2, v1
+; GFX900-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
; GFX900-SDAG-NEXT: s_endpgm
;
; GFX900-GISEL-LABEL: s_log10_f32:
@@ -192,15 +190,14 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
;
; GFX1100-SDAG-LABEL: s_log10_f32:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s1, 0x800000, s0
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s3, 0x800000, s2
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x411a209b, s1
-; GFX1100-SDAG-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX1100-SDAG-NEXT: s_cselect_b32 s1, 32, 0
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s0, s1
-; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x411a209b, s3
+; GFX1100-SDAG-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s2, s3
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -213,7 +210,6 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v2 :: v_dual_mov_b32 v2, 0
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX1100-SDAG-NEXT: s_endpgm
;
@@ -319,31 +315,32 @@ define amdgpu_kernel void @s_log10_f32(ptr addrspace(1) %out, float %in) {
define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in) {
; SI-SDAG-LABEL: s_log10_contract_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x3e9a209a
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; SI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; SI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s6, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s2, v1
; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x3e9a209a
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209a, v1
-; SI-SDAG-NEXT: v_fma_f32 v3, v1, s4, -v2
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x3284fbcf
-; SI-SDAG-NEXT: v_fma_f32 v3, v1, s4, v3
-; SI-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
+; SI-SDAG-NEXT: v_fma_f32 v3, v1, s0, -v2
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x3284fbcf
+; SI-SDAG-NEXT: v_fma_f32 v3, v1, s0, v3
+; SI-SDAG-NEXT: s_mov_b32 s0, 0x7f800000
; SI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
-; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s4
+; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s0
; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
; SI-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: s_log10_contract_f32:
@@ -376,34 +373,32 @@ define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in
;
; VI-SDAG-LABEL: s_log10_contract_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s0
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
-; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
-; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v1
-; VI-SDAG-NEXT: v_sub_f32_e32 v3, v1, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x369a84fb, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x3e9a2000, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x369a84fb, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v4, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a2000, v2
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v5, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s2
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; VI-SDAG-NEXT: v_sub_f32_e32 v2, v1, v0
-; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; VI-SDAG-NEXT: v_ldexp_f32 v0, s2, v0
+; VI-SDAG-NEXT: v_log_f32_e32 v3, v0
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: s_mov_b32 s0, 0x7f800000
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
+; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
+; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
+; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x369a84fb, v4
+; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3e9a2000, v5
+; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x369a84fb, v5
+; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
+; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
+; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3e9a2000, v4
+; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s0
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v4, vcc
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_sub_f32_e32 v2, v3, v2
; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
; VI-SDAG-NEXT: s_endpgm
;
@@ -440,30 +435,29 @@ define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in
;
; GFX900-SDAG-LABEL: s_log10_contract_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x800000
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0x411a209b
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
-; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
-; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v1
+; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, s3
+; GFX900-SDAG-NEXT: v_ldexp_f32 v2, s2, v2
+; GFX900-SDAG-NEXT: v_log_f32_e32 v2, v2
; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x3e9a209a
; GFX900-SDAG-NEXT: s_mov_b32 s3, 0x3284fbcf
-; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209a, v1
-; GFX900-SDAG-NEXT: v_fma_f32 v4, v1, s2, -v3
-; GFX900-SDAG-NEXT: v_fma_f32 v4, v1, s3, v4
-; GFX900-SDAG-NEXT: s_mov_b32 s2, 0x7f800000
+; GFX900-SDAG-NEXT: s_mov_b32 s4, 0x7f800000
+; GFX900-SDAG-NEXT: v_mul_f32_e32 v3, 0x3e9a209a, v2
+; GFX900-SDAG-NEXT: v_fma_f32 v4, v2, s2, -v3
+; GFX900-SDAG-NEXT: v_fma_f32 v4, v2, s3, v4
; GFX900-SDAG-NEXT: v_add_f32_e32 v3, v3, v4
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v1|, s2
-; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v1, v1, v3, vcc
-; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX900-SDAG-NEXT: global_store_dword v2, v0, s[0:1]
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v2|, s4
+; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GFX900-SDAG-NEXT: v_sub_f32_e32 v1, v2, v1
+; GFX900-SDAG-NEXT: global_store_dword v0, v1, s[0:1]
; GFX900-SDAG-NEXT: s_endpgm
;
; GFX900-GISEL-LABEL: s_log10_contract_f32:
@@ -495,15 +489,14 @@ define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in
;
; GFX1100-SDAG-LABEL: s_log10_contract_f32:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s1, 0x800000, s0
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s3, 0x800000, s2
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x411a209b, s1
-; GFX1100-SDAG-NEXT: s_and_b32 s1, s1, exec_lo
-; GFX1100-SDAG-NEXT: s_cselect_b32 s1, 32, 0
-; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s0, s1
-; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x411a209b, s3
+; GFX1100-SDAG-NEXT: s_and_b32 s3, s3, exec_lo
+; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s2, s3
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
@@ -516,7 +509,6 @@ define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in
; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_dual_cndmask_b32 v1, v1, v2 :: v_dual_mov_b32 v2, 0
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX1100-SDAG-NEXT: s_endpgm
;
@@ -624,38 +616,39 @@ define amdgpu_kernel void @s_log10_contract_f32(ptr addrspace(1) %out, float %in
define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in) {
; SI-SDAG-LABEL: s_log10_v2f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
-; SI-SDAG-NEXT: s_mov_b32 s8, 0x3e9a209a
+; SI-SDAG-NEXT: s_mov_b32 s8, 0x3284fbcf
; SI-SDAG-NEXT: s_mov_b32 s9, 0x7f800000
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s5, v0
-; SI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; SI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
-; SI-SDAG-NEXT: v_mov_b32_e32 v3, s2
-; SI-SDAG-NEXT: v_ldexp_f32_e32 v3, s5, v3
+; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s7, v0
+; SI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
+; SI-SDAG-NEXT: v_mov_b32_e32 v3, s0
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v3, s7, v3
; SI-SDAG-NEXT: v_log_f32_e32 v3, v3
; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s4, v0
-; SI-SDAG-NEXT: s_and_b64 s[6:7], vcc, exec
+; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
+; SI-SDAG-NEXT: s_mov_b32 s0, s4
+; SI-SDAG-NEXT: s_mov_b32 s1, s5
+; SI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-SDAG-NEXT: s_mov_b32 s7, 0x3e9a209a
; SI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3e9a209a, v3
-; SI-SDAG-NEXT: s_cselect_b32 s6, 32, 0
-; SI-SDAG-NEXT: s_mov_b32 s5, 0x3284fbcf
-; SI-SDAG-NEXT: v_fma_f32 v5, v3, s8, -v4
+; SI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
+; SI-SDAG-NEXT: v_fma_f32 v5, v3, s7, -v4
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT: v_mov_b32_e32 v1, s6
-; SI-SDAG-NEXT: v_fma_f32 v5, v3, s5, v5
-; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s4, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; SI-SDAG-NEXT: v_fma_f32 v5, v3, s8, v5
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s6, v1
; SI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
; SI-SDAG-NEXT: v_log_f32_e32 v5, v1
; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s9
; SI-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
; SI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v2
; SI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a209a, v5
-; SI-SDAG-NEXT: v_fma_f32 v3, v5, s8, -v2
-; SI-SDAG-NEXT: v_fma_f32 v3, v5, s5, v3
+; SI-SDAG-NEXT: v_fma_f32 v3, v5, s7, -v2
+; SI-SDAG-NEXT: v_fma_f32 v3, v5, s8, v3
; SI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
; SI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v5|, s9
; SI-SDAG-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
@@ -708,51 +701,51 @@ define amdgpu_kernel void @s_log10_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; VI-SDAG-LABEL: s_log10_v2f32:
; VI-SDAG: ; %bb.0:
; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x411a209b
+; VI-SDAG-NEXT: v_mov_b32_e32 v2, 0x800000
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, 0x411a209b
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v2
; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
; VI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s4
-; VI-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
-; VI-SDAG-NEXT: v_log_f32_e32 v3, v3
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
-; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
-; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v3
-; VI-SDAG-NEXT: v_sub_f32_e32 v5, v3, v4
-; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x369a84fb, v4
-; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x3e9a2000, v5
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; VI-SDAG-NEXT: v_ldexp_f32 v0, s3, v0
+; VI-SDAG-NEXT: v_log_f32_e32 v5, v0
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v3, vcc
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v2
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: v_and_b32_e32 v6, 0xfffff000, v5
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_sub_f32_e32 v7, v5, v6
+; VI-SDAG-NEXT: s_and_b64 s[0:1], vcc, exec
+; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x369a84fb, v6
+; VI-SDAG-NEXT: v_mul_f32_e32 v9, 0x3e9a2000, v7
+; VI-SDAG-NEXT: v_mul_f32_e32 v7, 0x369a84fb, v7
+; VI-SDAG-NEXT: s_cselect_b32 s0, 32, 0
+; VI-SDAG-NEXT: v_add_f32_e32 v7, v8, v7
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v3, vcc
+; VI-SDAG-NEXT: v_mov_b32_e32 v3, s0
+; VI-SDAG-NEXT: v_add_f32_e32 v7, v9, v7
+; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3e9a2000, v6
+; VI-SDAG-NEXT: v_ldexp_f32 v3, s2, v3
+; VI-SDAG-NEXT: v_add_f32_e32 v6, v6, v7
+; VI-SDAG-NEXT: v_log_f32_e32 v7, v3
+; VI-SDAG-NEXT: s_mov_b32 s3, 0x7f800000
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v5|, s3
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v3, v5, v6, vcc
+; VI-SDAG-NEXT: v_sub_f32_e32 v3, v3, v4
+; VI-SDAG-NEXT: v_and_b32_e32 v4, 0xfffff000, v7
+; VI-SDAG-NEXT: v_sub_f32_e32 v5, v7, v4
+; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x3e9a2000, v5
; VI-SDAG-NEXT: v_mul_f32_e32 v5, 0x369a84fb, v5
-; VI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
+; VI-SDAG-NEXT: v_mul_f32_e32 v8, 0x369a84fb, v4
+; VI-SDAG-NEXT: v_add_f32_e32 v5, v8, v5
; VI-SDAG-NEXT: v_add_f32_e32 v5, v6, v5
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s4
; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3e9a2000, v4
-; VI-SDAG-NEXT: v_add_f32_e32 v5, v7, v5
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
; VI-SDAG-NEXT: v_add_f32_e32 v4, v4, v5
-; VI-SDAG-NEXT: v_log_f32_e32 v5, v1
-; VI-SDAG-NEXT: s_mov_b32 s3, 0x7f800000
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v3|, s3
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v4, vcc
-; VI-SDAG-NEXT: v_sub_f32_e32 v1, v1, v2
-; VI-SDAG-NEXT: v_and_b32_e32 v2, 0xfffff000, v5
-; VI-SDAG-NEXT: v_sub_f32_e32 v3, v5, v2
-; VI-SDAG-NEXT: v_mul_f32_e32 v4, 0x3e9a2000, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v3, 0x369a84fb, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v6, 0x369a84fb, v2
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v6, v3
-; VI-SDAG-NEXT: v_add_f32_e32 v3, v4, v3
-; VI-SDAG-NEXT: v_mul_f32_e32 v2, 0x3e9a2000, v2
-; VI-SDAG-NEXT: v_add_f32_e32 v2, v2, v3
-; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v5|, s3
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, v5, v2, vcc
-; VI-SDAG-NEXT: v_sub_f32_e32 v0, v2, v0
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; VI-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-SDAG-NEXT: v_cmp_lt_f32_e64 vcc, |v7|, s3
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, v7, v4, vcc
+; VI-SDAG-NEXT: v_sub_f32_e32 v2, v4, v2
+; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
; VI-GISEL-LABEL: s_log10_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
index b2ec9534aa848..4a02f4318a70f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.log2.ll
@@ -16,19 +16,18 @@
define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
; SI-SDAG-LABEL: s_log2_f32:
; SI-SDAG: ; %bb.0:
-; SI-SDAG-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42000000
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; SI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; SI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
+; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; SI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; SI-SDAG-NEXT: s_cselect_b32 s4, 32, 0
; SI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; SI-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s6, v1
+; SI-SDAG-NEXT: v_mov_b32_e32 v1, s4
+; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s2, v1
; SI-SDAG-NEXT: v_log_f32_e32 v1, v1
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
; SI-SDAG-NEXT: s_mov_b32 s2, -1
; SI-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
; SI-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -55,21 +54,20 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
;
; VI-SDAG-LABEL: s_log2_f32:
; VI-SDAG: ; %bb.0:
-; VI-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; VI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42000000
; VI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; VI-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; VI-SDAG-NEXT: s_cselect_b32 s2, 32, 0
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
-; VI-SDAG-NEXT: v_log_f32_e32 v1, v1
-; VI-SDAG-NEXT: v_sub_f32_e32 v2, v1, v0
+; VI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; VI-SDAG-NEXT: v_ldexp_f32 v0, s2, v0
+; VI-SDAG-NEXT: v_log_f32_e32 v3, v0
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v2, 0, v1, vcc
; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_sub_f32_e32 v2, v3, v2
; VI-SDAG-NEXT: flat_store_dword v[0:1], v2
; VI-SDAG-NEXT: s_endpgm
;
@@ -94,18 +92,17 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
;
; GFX900-SDAG-LABEL: s_log2_f32:
; GFX900-SDAG: ; %bb.0:
-; GFX900-SDAG-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX900-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX900-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, 0x42000000
; GFX900-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX900-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; GFX900-SDAG-NEXT: s_and_b64 s[2:3], vcc, exec
-; GFX900-SDAG-NEXT: s_cselect_b32 s2, 32, 0
+; GFX900-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GFX900-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
+; GFX900-SDAG-NEXT: s_cselect_b32 s3, 32, 0
; GFX900-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s2
-; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s6, v1
+; GFX900-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; GFX900-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
; GFX900-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX900-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
; GFX900-SDAG-NEXT: global_store_dword v2, v0, s[0:1]
@@ -131,20 +128,19 @@ define amdgpu_kernel void @s_log2_f32(ptr addrspace(1) %out, float %in) {
;
; GFX1100-SDAG-LABEL: s_log2_f32:
; GFX1100-SDAG: ; %bb.0:
-; GFX1100-SDAG-NEXT: s_load_b32 s2, s[4:5], 0x2c
+; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v2, 0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s0, 0x800000, s2
-; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x42000000, s0
-; GFX1100-SDAG-NEXT: s_and_b32 s0, s0, exec_lo
+; GFX1100-SDAG-NEXT: v_cmp_gt_f32_e64 s3, 0x800000, s2
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX1100-SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 0x42000000, s3
+; GFX1100-SDAG-NEXT: s_and_b32 s3, s3, exec_lo
; GFX1100-SDAG-NEXT: s_cselect_b32 s3, 32, 0
-; GFX1100-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; GFX1100-SDAG-NEXT: v_ldexp_f32 v1, s2, s3
+; GFX1100-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-SDAG-NEXT: v_log_f32_e32 v1, v1
; GFX1100-SDAG-NEXT: s_waitcnt_depctr depctr_va_vdst(0)
; GFX1100-SDAG-NEXT: v_sub_f32_e32 v0, v1, v0
-; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX1100-SDAG-NEXT: s_endpgm
;
@@ -221,6 +217,8 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; SI-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-SDAG-NEXT: v_mov_b32_e32 v0, 0x800000
; SI-SDAG-NEXT: v_mov_b32_e32 v1, 0x42000000
+; SI-SDAG-NEXT: s_mov_b32 s7, 0xf000
+; SI-SDAG-NEXT: s_mov_b32 s6, -1
; SI-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; SI-SDAG-NEXT: v_cmp_lt_f32_e32 vcc, s3, v0
; SI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
@@ -236,11 +234,11 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; SI-SDAG-NEXT: v_ldexp_f32_e32 v1, s2, v1
; SI-SDAG-NEXT: v_log_f32_e32 v3, v3
; SI-SDAG-NEXT: v_log_f32_e32 v4, v1
-; SI-SDAG-NEXT: s_mov_b32 s3, 0xf000
-; SI-SDAG-NEXT: s_mov_b32 s2, -1
+; SI-SDAG-NEXT: s_mov_b32 s4, s0
+; SI-SDAG-NEXT: s_mov_b32 s5, s1
; SI-SDAG-NEXT: v_sub_f32_e32 v1, v3, v2
; SI-SDAG-NEXT: v_sub_f32_e32 v0, v4, v0
-; SI-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-SDAG-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-SDAG-NEXT: s_endpgm
;
; SI-GISEL-LABEL: s_log2_v2f32:
@@ -283,16 +281,16 @@ define amdgpu_kernel void @s_log2_v2f32(ptr addrspace(1) %out, <2 x float> %in)
; VI-SDAG-NEXT: s_and_b64 s[4:5], vcc, exec
; VI-SDAG-NEXT: v_ldexp_f32 v3, s3, v3
; VI-SDAG-NEXT: s_cselect_b32 s3, 32, 0
-; VI-SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v1, vcc
-; VI-SDAG-NEXT: v_mov_b32_e32 v1, s3
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s3
+; VI-SDAG-NEXT: v_ldexp_f32 v0, s2, v0
; VI-SDAG-NEXT: v_log_f32_e32 v3, v3
-; VI-SDAG-NEXT: v_ldexp_f32 v1, s2, v1
-; VI-SDAG-NEXT: v_log_f32_e32 v4, v1
-; VI-SDAG-NEXT: v_sub_f32_e32 v1, v3, v2
-; VI-SDAG-NEXT: v_mov_b32_e32 v3, s1
-; VI-SDAG-NEXT: v_sub_f32_e32 v0, v4, v0
-; VI-SDAG-NEXT: v_mov_b32_e32 v2, s0
-; VI-SDAG-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-SDAG-NEXT: v_log_f32_e32 v5, v0
+; VI-SDAG-NEXT: v_cndmask_b32_e32 v4, 0, v1, vcc
+; VI-SDAG-NEXT: v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT: v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT: v_sub_f32_e32 v3, v3, v2
+; VI-SDAG-NEXT: v_sub_f32_e32 v2, v5, v4
+; VI-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-SDAG-NEXT: s_endpgm
;
; VI-GISEL-LABEL: s_log2_v2f32:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.round.ll b/llvm/test/CodeGen/AMDGPU/llvm.round.ll
index 802ffc1e7ea7b..c9bfd31de9785 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.round.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.round.ll
@@ -9,63 +9,64 @@
define amdgpu_kernel void @round_f32(ptr addrspace(1) %out, float %x) #0 {
; GFX6-LABEL: round_f32:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_trunc_f32_e32 v0, s6
-; GFX6-NEXT: v_sub_f32_e32 v1, s6, v0
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, 0.5
-; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s[4:5]
-; GFX6-NEXT: s_brev_b32 s4, -2
-; GFX6-NEXT: v_mov_b32_e32 v2, s6
-; GFX6-NEXT: v_bfi_b32 v1, s4, v1, v2
+; GFX6-NEXT: v_trunc_f32_e32 v0, s2
+; GFX6-NEXT: v_sub_f32_e32 v1, s2, v0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, 0.5
+; GFX6-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s[0:1]
+; GFX6-NEXT: s_brev_b32 s0, -2
+; GFX6-NEXT: v_mov_b32_e32 v2, s2
+; GFX6-NEXT: v_bfi_b32 v1, s0, v1, v2
; GFX6-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: round_f32:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_trunc_f32_e32 v0, s6
-; GFX8-NEXT: v_sub_f32_e32 v1, s6, v0
-; GFX8-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, 0.5
-; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s[4:5]
-; GFX8-NEXT: s_brev_b32 s4, -2
-; GFX8-NEXT: v_mov_b32_e32 v2, s6
-; GFX8-NEXT: v_bfi_b32 v1, s4, v1, v2
+; GFX8-NEXT: v_trunc_f32_e32 v0, s2
+; GFX8-NEXT: v_sub_f32_e32 v1, s2, v0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, 0.5
+; GFX8-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s[0:1]
+; GFX8-NEXT: s_brev_b32 s0, -2
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_bfi_b32 v1, s0, v1, v2
; GFX8-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: round_f32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_trunc_f32_e32 v0, s6
-; GFX9-NEXT: v_sub_f32_e32 v1, s6, v0
-; GFX9-NEXT: v_cmp_ge_f32_e64 s[4:5], |v1|, 0.5
-; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s[4:5]
-; GFX9-NEXT: s_brev_b32 s4, -2
-; GFX9-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-NEXT: v_bfi_b32 v1, s4, v1, v2
+; GFX9-NEXT: v_trunc_f32_e32 v0, s2
+; GFX9-NEXT: v_sub_f32_e32 v1, s2, v0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: v_cmp_ge_f32_e64 s[0:1], |v1|, 0.5
+; GFX9-NEXT: v_cndmask_b32_e64 v1, 0, 1.0, s[0:1]
+; GFX9-NEXT: s_brev_b32 s0, -2
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_bfi_b32 v1, s0, v1, v2
; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: round_f32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_trunc_f32_e32 v0, s2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -769,67 +770,65 @@ define amdgpu_kernel void @round_v8f32(ptr addrspace(1) %out, <8 x float> %in) #
define amdgpu_kernel void @round_f16(ptr addrspace(1) %out, i32 %x.arg) #0 {
; GFX6-LABEL: round_f16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s0
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX6-NEXT: s_brev_b32 s2, -2
; GFX6-NEXT: v_trunc_f32_e32 v1, v0
; GFX6-NEXT: v_sub_f32_e32 v2, v0, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v2|, 0.5
-; GFX6-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s[2:3]
-; GFX6-NEXT: s_brev_b32 s2, -2
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v2|, 0.5
+; GFX6-NEXT: v_cndmask_b32_e64 v2, 0, 1.0, s[4:5]
; GFX6-NEXT: v_bfi_b32 v0, s2, v2, v0
; GFX6-NEXT: v_add_f32_e32 v0, v1, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_short v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: round_f16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: v_mov_b32_e32 v0, 0x3c00
-; GFX8-NEXT: s_movk_i32 s4, 0x7fff
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: v_mov_b32_e32 v2, 0x3c00
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_trunc_f16_e32 v1, s6
-; GFX8-NEXT: v_sub_f16_e32 v2, s6, v1
-; GFX8-NEXT: v_cmp_ge_f16_e64 vcc, |v2|, 0.5
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v2, s6
-; GFX8-NEXT: v_bfi_b32 v0, s4, v0, v2
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_add_f16_e32 v0, v1, v0
-; GFX8-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX8-NEXT: v_trunc_f16_e32 v0, s2
+; GFX8-NEXT: v_sub_f16_e32 v1, s2, v0
+; GFX8-NEXT: v_cmp_ge_f16_e64 vcc, |v1|, 0.5
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; GFX8-NEXT: s_movk_i32 s0, 0x7fff
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_bfi_b32 v1, s0, v1, v2
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX8-NEXT: buffer_store_short v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: round_f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x3c00
-; GFX9-NEXT: s_movk_i32 s4, 0x7fff
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v2, 0x3c00
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_trunc_f16_e32 v1, s6
-; GFX9-NEXT: v_sub_f16_e32 v2, s6, v1
-; GFX9-NEXT: v_cmp_ge_f16_e64 vcc, |v2|, 0.5
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v2, s6
-; GFX9-NEXT: v_bfi_b32 v0, s4, v0, v2
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_add_f16_e32 v0, v1, v0
-; GFX9-NEXT: buffer_store_short v0, off, s[0:3], 0
+; GFX9-NEXT: v_trunc_f16_e32 v0, s2
+; GFX9-NEXT: v_sub_f16_e32 v1, s2, v0
+; GFX9-NEXT: v_cmp_ge_f16_e64 vcc, |v1|, 0.5
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; GFX9-NEXT: s_movk_i32 s0, 0x7fff
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: v_bfi_b32 v1, s0, v1, v2
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: buffer_store_short v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: round_f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: v_trunc_f16_e32 v0.l, s2
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s2
@@ -848,9 +847,7 @@ define amdgpu_kernel void @round_f16(ptr addrspace(1) %out, i32 %x.arg) #0 {
;
; GFX11-FAKE16-LABEL: round_f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_trunc_f16_e32 v0, s2
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
@@ -902,96 +899,94 @@ define amdgpu_kernel void @round_f16(ptr addrspace(1) %out, i32 %x.arg) #0 {
define amdgpu_kernel void @round_v2f16(ptr addrspace(1) %out, i32 %in.arg) #0 {
; GFX6-LABEL: round_v2f16:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s0, s[4:5], 0xb
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_brev_b32 s6, -2
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_lshr_b32 s1, s0, 16
-; GFX6-NEXT: v_cvt_f32_f16_e32 v1, s1
-; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s0
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_brev_b32 s4, -2
+; GFX6-NEXT: s_mov_b32 s3, 0xf000
+; GFX6-NEXT: s_lshr_b32 s4, s2, 16
+; GFX6-NEXT: v_cvt_f32_f16_e32 v1, s4
+; GFX6-NEXT: v_cvt_f32_f16_e32 v0, s2
+; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_trunc_f32_e32 v3, v1
; GFX6-NEXT: v_sub_f32_e32 v5, v1, v3
; GFX6-NEXT: v_trunc_f32_e32 v2, v0
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v5|, 0.5
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v5|, 0.5
; GFX6-NEXT: v_sub_f32_e32 v4, v0, v2
-; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1.0, s[2:3]
-; GFX6-NEXT: v_bfi_b32 v1, s4, v5, v1
-; GFX6-NEXT: v_cmp_ge_f32_e64 s[2:3], |v4|, 0.5
+; GFX6-NEXT: v_cndmask_b32_e64 v5, 0, 1.0, s[4:5]
+; GFX6-NEXT: v_bfi_b32 v1, s6, v5, v1
+; GFX6-NEXT: v_cmp_ge_f32_e64 s[4:5], |v4|, 0.5
; GFX6-NEXT: v_add_f32_e32 v1, v3, v1
-; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1.0, s[2:3]
-; GFX6-NEXT: v_bfi_b32 v0, s4, v3, v0
+; GFX6-NEXT: v_cndmask_b32_e64 v3, 0, 1.0, s[4:5]
+; GFX6-NEXT: v_bfi_b32 v0, s6, v3, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v1, v1
; GFX6-NEXT: v_add_f32_e32 v0, v2, v0
; GFX6-NEXT: v_cvt_f16_f32_e32 v0, v0
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: s_mov_b32 s2, -1
; GFX6-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: round_v2f16:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: v_mov_b32_e32 v0, 0x3c00
-; GFX8-NEXT: s_movk_i32 s5, 0x7fff
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: v_mov_b32_e32 v2, 0x3c00
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s4, s6, 16
-; GFX8-NEXT: v_trunc_f16_e32 v1, s4
-; GFX8-NEXT: v_sub_f16_e32 v2, s4, v1
-; GFX8-NEXT: v_cmp_ge_f16_e64 vcc, |v2|, 0.5
-; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v3, s4
-; GFX8-NEXT: v_bfi_b32 v2, s5, v2, v3
-; GFX8-NEXT: v_add_f16_sdwa v1, v1, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX8-NEXT: v_trunc_f16_e32 v2, s6
-; GFX8-NEXT: v_sub_f16_e32 v3, s6, v2
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_lshr_b32 s0, s2, 16
+; GFX8-NEXT: v_trunc_f16_e32 v0, s0
+; GFX8-NEXT: v_sub_f16_e32 v1, s0, v0
+; GFX8-NEXT: v_cmp_ge_f16_e64 vcc, |v1|, 0.5
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; GFX8-NEXT: s_movk_i32 s1, 0x7fff
+; GFX8-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-NEXT: v_bfi_b32 v1, s1, v1, v3
+; GFX8-NEXT: v_add_f16_sdwa v0, v0, v1 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX8-NEXT: v_trunc_f16_e32 v1, s2
+; GFX8-NEXT: v_sub_f16_e32 v3, s2, v1
; GFX8-NEXT: v_cmp_ge_f16_e64 vcc, |v3|, 0.5
-; GFX8-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX8-NEXT: v_mov_b32_e32 v3, s6
-; GFX8-NEXT: v_bfi_b32 v0, s5, v0, v3
-; GFX8-NEXT: v_add_f16_e32 v0, v2, v0
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX8-NEXT: v_mov_b32_e32 v3, s2
+; GFX8-NEXT: v_bfi_b32 v2, s1, v2, v3
+; GFX8-NEXT: v_add_f16_e32 v1, v1, v2
+; GFX8-NEXT: v_or_b32_e32 v0, v1, v0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: round_v2f16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: v_mov_b32_e32 v0, 0x3c00
-; GFX9-NEXT: s_movk_i32 s5, 0x7fff
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: v_mov_b32_e32 v2, 0x3c00
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s6, 16
-; GFX9-NEXT: v_trunc_f16_e32 v1, s4
-; GFX9-NEXT: v_sub_f16_e32 v2, s4, v1
-; GFX9-NEXT: v_cmp_ge_f16_e64 vcc, |v2|, 0.5
-; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, s4
-; GFX9-NEXT: v_bfi_b32 v2, s5, v2, v3
-; GFX9-NEXT: v_add_f16_e32 v1, v1, v2
-; GFX9-NEXT: v_trunc_f16_e32 v2, s6
-; GFX9-NEXT: v_sub_f16_e32 v3, s6, v2
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_lshr_b32 s0, s2, 16
+; GFX9-NEXT: v_trunc_f16_e32 v0, s0
+; GFX9-NEXT: v_sub_f16_e32 v1, s0, v0
+; GFX9-NEXT: v_cmp_ge_f16_e64 vcc, |v1|, 0.5
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: v_cndmask_b32_e32 v1, 0, v2, vcc
+; GFX9-NEXT: s_movk_i32 s1, 0x7fff
+; GFX9-NEXT: v_mov_b32_e32 v3, s0
+; GFX9-NEXT: v_bfi_b32 v1, s1, v1, v3
+; GFX9-NEXT: v_add_f16_e32 v0, v0, v1
+; GFX9-NEXT: v_trunc_f16_e32 v1, s2
+; GFX9-NEXT: v_sub_f16_e32 v3, s2, v1
; GFX9-NEXT: v_cmp_ge_f16_e64 vcc, |v3|, 0.5
-; GFX9-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GFX9-NEXT: v_mov_b32_e32 v3, s6
-; GFX9-NEXT: v_bfi_b32 v0, s5, v0, v3
-; GFX9-NEXT: v_add_f16_e32 v0, v2, v0
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_pack_b32_f16 v0, v0, v1
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
+; GFX9-NEXT: v_bfi_b32 v2, s1, v2, v3
+; GFX9-NEXT: v_add_f16_e32 v1, v1, v2
+; GFX9-NEXT: v_pack_b32_f16 v0, v1, v0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: round_v2f16:
; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_lshr_b32 s3, s2, 16
; GFX11-TRUE16-NEXT: v_trunc_f16_e32 v0.l, s2
@@ -1002,12 +997,12 @@ define amdgpu_kernel void @round_v2f16(ptr addrspace(1) %out, i32 %in.arg) #0 {
; GFX11-TRUE16-NEXT: v_sub_f16_e32 v1.h, s3, v0.h
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_cmp_ge_f16_e64 s6, |v1.l|, 0.5
-; GFX11-TRUE16-NEXT: v_cmp_ge_f16_e64 s7, |v1.h|, 0.5
+; GFX11-TRUE16-NEXT: v_cmp_ge_f16_e64 s4, |v1.l|, 0.5
+; GFX11-TRUE16-NEXT: v_cmp_ge_f16_e64 s5, |v1.h|, 0.5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, s2
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 0x3c00, s6
-; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, 0, 0x3c00, s7
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v2.l, 0, 0x3c00, s4
+; GFX11-TRUE16-NEXT: v_cndmask_b16 v3.l, 0, 0x3c00, s5
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_bfi_b32 v1, 0x7fff, v2, v1
; GFX11-TRUE16-NEXT: v_bfi_b32 v2, 0x7fff, v3, v4
@@ -1019,9 +1014,7 @@ define amdgpu_kernel void @round_v2f16(ptr addrspace(1) %out, i32 %in.arg) #0 {
;
; GFX11-FAKE16-LABEL: round_v2f16:
; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_lshr_b32 s3, s2, 16
; GFX11-FAKE16-NEXT: v_trunc_f16_e32 v1, s2
diff --git a/llvm/test/CodeGen/AMDGPU/load-select-ptr.ll b/llvm/test/CodeGen/AMDGPU/load-select-ptr.ll
index 3fedd68edaea2..c295fd65e646e 100644
--- a/llvm/test/CodeGen/AMDGPU/load-select-ptr.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-select-ptr.ll
@@ -145,14 +145,12 @@ define amdgpu_kernel void @select_ptr_crash_i64_local_offsets(i32 %tmp, ptr addr
define amdgpu_kernel void @sample_test(ptr addrspace(1) %dest, ptr addrspace(1) %sourceA, ptr addrspace(1) %sourceB, i1 %tobool.not.i) #0 {
; GCN-LABEL: sample_test:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x18
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x10
+; GCN-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x0
; GCN-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_bitcmp1_b32 s6, 0
; GCN-NEXT: v_mov_b32_e32 v0, s3
; GCN-NEXT: v_add_u32_e32 v3, vcc, s2, v2
+; GCN-NEXT: s_bitcmp1_b32 s6, 0
; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
@@ -182,14 +180,12 @@ entry:
define amdgpu_kernel void @constant_is_not_uniform(ptr addrspace(1) %dest, ptr addrspace(4) %sourceA, ptr addrspace(4) %sourceB, i1 %tobool.not.i) #0 {
; GCN-LABEL: constant_is_not_uniform:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s6, s[4:5], 0x18
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x10
+; GCN-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x0
; GCN-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_bitcmp1_b32 s6, 0
; GCN-NEXT: v_mov_b32_e32 v0, s3
; GCN-NEXT: v_add_u32_e32 v3, vcc, s2, v2
+; GCN-NEXT: s_bitcmp1_b32 s6, 0
; GCN-NEXT: v_addc_u32_e32 v0, vcc, 0, v0, vcc
; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: s_cselect_b64 vcc, -1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
index b0da72184f050..4258ceae8fa7d 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomicrmw-fadd.ll
@@ -8157,34 +8157,34 @@ define void @local_atomic_fadd_noret_v2bf16__ofset(ptr addrspace(3) %ptr, <2 x b
define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3) %ptrf, i32 %idx) {
; GFX12-LABEL: local_ds_fadd:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX12-NEXT: s_mov_b32 s6, exec_lo
+; GFX12-NEXT: s_load_b128 s[4:7], s[4:5], 0x0
+; GFX12-NEXT: s_mov_b32 s2, exec_lo
; GFX12-NEXT: ; implicit-def: $vgpr1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_add_co_i32 s1, s3, 4
+; GFX12-NEXT: s_add_co_i32 s1, s7, 4
; GFX12-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX12-NEXT: s_cbranch_execz .LBB28_2
; GFX12-NEXT: ; %bb.1:
-; GFX12-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX12-NEXT: s_lshl_b32 s3, s1, 3
+; GFX12-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX12-NEXT: s_lshl_b32 s2, s1, 3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1
; GFX12-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX12-NEXT: s_wait_dscnt 0x0
; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: .LBB28_2:
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX12-NEXT: s_mov_b32 s7, exec_lo
-; GFX12-NEXT: v_readfirstlane_b32 s3, v1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_readfirstlane_b32 s2, v1
; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
-; GFX12-NEXT: s_mov_b32 s6, exec_lo
+; GFX12-NEXT: s_mov_b32 s3, exec_lo
; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX12-NEXT: s_cbranch_execz .LBB28_4
; GFX12-NEXT: ; %bb.3:
@@ -8200,28 +8200,28 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX12-NEXT: global_inv scope:SCOPE_SE
; GFX12-NEXT: .LBB28_4:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX12-NEXT: s_mov_b32 s1, exec_lo
; GFX12-NEXT: s_brev_b32 s0, 1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX12-NEXT: v_add_f32_e32 v0, s3, v0
+; GFX12-NEXT: v_add_f32_e32 v0, s2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e64 v1, v0, s3, vcc_lo
+; GFX12-NEXT: v_cndmask_b32_e64 v1, v0, s2, vcc_lo
; GFX12-NEXT: ; implicit-def: $vgpr0
; GFX12-NEXT: .LBB28_5: ; %ComputeLoop
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_ctz_i32_b32 s3, s1
+; GFX12-NEXT: s_ctz_i32_b32 s2, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_readlane_b32 s6, v1, s3
-; GFX12-NEXT: v_writelane_b32 v0, s0, s3
-; GFX12-NEXT: s_lshl_b32 s3, 1, s3
+; GFX12-NEXT: v_readlane_b32 s3, v1, s2
+; GFX12-NEXT: v_writelane_b32 v0, s0, s2
+; GFX12-NEXT: s_lshl_b32 s2, 1, s2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 s1, s1, s3
-; GFX12-NEXT: s_add_f32 s0, s0, s6
+; GFX12-NEXT: s_and_not1_b32 s1, s1, s2
+; GFX12-NEXT: s_add_f32 s0, s0, s3
; GFX12-NEXT: s_cbranch_scc1 .LBB28_5
; GFX12-NEXT: ; %bb.6: ; %ComputeEnd
; GFX12-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
@@ -8233,7 +8233,7 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX12-NEXT: s_xor_b32 s1, exec_lo, s1
; GFX12-NEXT: s_cbranch_execz .LBB28_8
; GFX12-NEXT: ; %bb.7:
-; GFX12-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v2, s0
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX12-NEXT: s_wait_dscnt 0x0
@@ -8241,55 +8241,53 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX12-NEXT: .LBB28_8:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s2, v1
+; GFX12-NEXT: v_readfirstlane_b32 s0, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s0, v0
+; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s0, vcc_lo
+; GFX12-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX12-NEXT: s_endpgm
;
; GFX942-LABEL: local_ds_fadd:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX942-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX942-NEXT: s_mov_b64 s[0:1], exec
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_add_i32 s3, s3, 4
+; GFX942-NEXT: s_add_i32 s7, s7, 4
; GFX942-NEXT: ; implicit-def: $vgpr1
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB28_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: s_lshl_b32 s8, s3, 3
+; GFX942-NEXT: s_lshl_b32 s8, s7, 3
; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX942-NEXT: v_mov_b32_e32 v2, s8
; GFX942-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: .LBB28_2:
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_mov_b64 s[8:9], exec
; GFX942-NEXT: v_readfirstlane_b32 s10, v1
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
; GFX942-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
+; GFX942-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
; GFX942-NEXT: s_cbranch_execz .LBB28_4
; GFX942-NEXT: ; %bb.3:
; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX942-NEXT: s_lshl_b32 s0, s3, 4
+; GFX942-NEXT: s_lshl_b32 s0, s7, 4
; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX942-NEXT: v_mov_b32_e32 v2, s0
; GFX942-NEXT: ds_add_f32 v2, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: .LBB28_4:
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX942-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX942-NEXT: v_add_f32_e32 v0, s10, v0
@@ -8300,14 +8298,14 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX942-NEXT: ; implicit-def: $vgpr0
; GFX942-NEXT: .LBB28_5: ; %ComputeLoop
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX942-NEXT: v_readfirstlane_b32 s6, v1
-; GFX942-NEXT: s_mov_b32 m0, s3
-; GFX942-NEXT: v_readlane_b32 s8, v2, s3
-; GFX942-NEXT: v_writelane_b32 v0, s6, m0
-; GFX942-NEXT: s_lshl_b64 s[6:7], 1, s3
-; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]
-; GFX942-NEXT: v_add_f32_e32 v1, s8, v1
+; GFX942-NEXT: s_ff1_i32_b64 s2, s[0:1]
+; GFX942-NEXT: v_readfirstlane_b32 s3, v1
+; GFX942-NEXT: s_mov_b32 m0, s2
+; GFX942-NEXT: v_readlane_b32 s7, v2, s2
+; GFX942-NEXT: v_writelane_b32 v0, s3, m0
+; GFX942-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX942-NEXT: v_add_f32_e32 v1, s7, v1
; GFX942-NEXT: s_cbranch_scc1 .LBB28_5
; GFX942-NEXT: ; %bb.6: ; %ComputeEnd
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -8318,41 +8316,39 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX942-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX942-NEXT: s_cbranch_execz .LBB28_8
; GFX942-NEXT: ; %bb.7:
-; GFX942-NEXT: v_mov_b32_e32 v2, s2
+; GFX942-NEXT: v_mov_b32_e32 v2, s6
; GFX942-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: .LBB28_8:
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942-NEXT: v_readfirstlane_b32 s2, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v2
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_add_f32_e32 v0, s2, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, s2
+; GFX942-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX942-NEXT: v_mov_b32_e32 v2, s0
; GFX942-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX942-NEXT: global_store_dword v1, v0, s[4:5]
; GFX942-NEXT: s_endpgm
;
; GFX11-LABEL: local_ds_fadd:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX11-NEXT: s_mov_b32 s6, exec_lo
+; GFX11-NEXT: s_load_b128 s[4:7], s[4:5], 0x0
+; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_add_i32 s1, s3, 4
+; GFX11-NEXT: s_add_i32 s1, s7, 4
; GFX11-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-NEXT: s_cbranch_execz .LBB28_2
; GFX11-NEXT: ; %bb.1:
-; GFX11-NEXT: s_bcnt1_i32_b32 s3, s6
+; GFX11-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX11-NEXT: s_lshl_b32 s3, s1, 3
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX11-NEXT: s_lshl_b32 s2, s1, 3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1
; GFX11-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
@@ -8360,9 +8356,9 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-NEXT: s_mov_b32 s7, exec_lo
-; GFX11-NEXT: v_readfirstlane_b32 s3, v1
+; GFX11-NEXT: v_readfirstlane_b32 s2, v1
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
-; GFX11-NEXT: s_mov_b32 s6, exec_lo
+; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX11-NEXT: s_cbranch_execz .LBB28_4
; GFX11-NEXT: ; %bb.3:
@@ -8376,25 +8372,25 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: .LBB28_4:
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_bfrev_b32_e32 v1, 1
; GFX11-NEXT: s_mov_b32 s0, exec_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX11-NEXT: v_add_f32_e32 v0, s3, v0
+; GFX11-NEXT: v_add_f32_e32 v0, s2, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, s3, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, s2, vcc_lo
; GFX11-NEXT: ; implicit-def: $vgpr0
; GFX11-NEXT: .LBB28_5: ; %ComputeLoop
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_ctz_i32_b32 s1, s0
-; GFX11-NEXT: v_readfirstlane_b32 s3, v1
+; GFX11-NEXT: v_readfirstlane_b32 s2, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readlane_b32 s6, v2, s1
-; GFX11-NEXT: v_writelane_b32 v0, s3, s1
+; GFX11-NEXT: v_readlane_b32 s3, v2, s1
+; GFX11-NEXT: v_writelane_b32 v0, s2, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_add_f32_e32 v1, s6, v1
+; GFX11-NEXT: v_add_f32_e32 v1, s3, v1
; GFX11-NEXT: s_lshl_b32 s1, 1, s1
; GFX11-NEXT: s_and_not1_b32 s0, s0, s1
; GFX11-NEXT: s_cbranch_scc1 .LBB28_5
@@ -8407,37 +8403,35 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: s_cbranch_execz .LBB28_8
; GFX11-NEXT: ; %bb.7:
-; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: v_mov_b32_e32 v2, s6
; GFX11-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: buffer_gl0_inv
; GFX11-NEXT: .LBB28_8:
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s0, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, s0, vcc_lo
+; GFX11-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX11-NEXT: s_endpgm
;
; GFX10-LABEL: local_ds_fadd:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
+; GFX10-NEXT: s_mov_b32 s2, exec_lo
; GFX10-NEXT: ; implicit-def: $vgpr1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_add_i32 s1, s3, 4
+; GFX10-NEXT: s_add_i32 s1, s7, 4
; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB28_2
; GFX10-NEXT: ; %bb.1:
-; GFX10-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX10-NEXT: s_lshl_b32 s3, s1, 3
-; GFX10-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX10-NEXT: s_lshl_b32 s2, s1, 3
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX10-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -8446,10 +8440,10 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX10-NEXT: s_mov_b32 s7, exec_lo
-; GFX10-NEXT: v_readfirstlane_b32 s3, v1
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v2
-; GFX10-NEXT: s_and_saveexec_b32 s6, s0
+; GFX10-NEXT: s_and_saveexec_b32 s3, s0
; GFX10-NEXT: s_cbranch_execz .LBB28_4
; GFX10-NEXT: ; %bb.3:
; GFX10-NEXT: s_bcnt1_i32_b32 s0, s7
@@ -8463,21 +8457,21 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX10-NEXT: buffer_gl0_inv
; GFX10-NEXT: .LBB28_4:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT: v_bfrev_b32_e32 v1, 1
; GFX10-NEXT: s_mov_b32 s0, exec_lo
; GFX10-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX10-NEXT: v_add_f32_e32 v0, s3, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, s3, vcc_lo
+; GFX10-NEXT: v_add_f32_e32 v0, s2, v0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, s2, vcc_lo
; GFX10-NEXT: ; implicit-def: $vgpr0
; GFX10-NEXT: .LBB28_5: ; %ComputeLoop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_ff1_i32_b32 s1, s0
-; GFX10-NEXT: v_readfirstlane_b32 s3, v1
-; GFX10-NEXT: v_readlane_b32 s6, v2, s1
-; GFX10-NEXT: v_writelane_b32 v0, s3, s1
-; GFX10-NEXT: v_add_f32_e32 v1, s6, v1
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10-NEXT: v_readlane_b32 s3, v2, s1
+; GFX10-NEXT: v_writelane_b32 v0, s2, s1
+; GFX10-NEXT: v_add_f32_e32 v1, s3, v1
; GFX10-NEXT: s_lshl_b32 s1, 1, s1
; GFX10-NEXT: s_andn2_b32 s0, s0, s1
; GFX10-NEXT: s_cbranch_scc1 .LBB28_5
@@ -8489,7 +8483,7 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX10-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX10-NEXT: s_cbranch_execz .LBB28_8
; GFX10-NEXT: ; %bb.7:
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: v_mov_b32_e32 v2, s6
; GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; GFX10-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
@@ -8497,54 +8491,52 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX10-NEXT: .LBB28_8:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX10-NEXT: v_readfirstlane_b32 s2, v2
+; GFX10-NEXT: v_readfirstlane_b32 s0, v2
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_add_f32_e32 v0, s2, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s0, vcc_lo
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX90A-LABEL: local_ds_fadd:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_add_i32 s3, s3, 4
+; GFX90A-NEXT: s_add_i32 s7, s7, 4
; GFX90A-NEXT: ; implicit-def: $vgpr1
-; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB28_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: s_lshl_b32 s8, s3, 3
+; GFX90A-NEXT: s_lshl_b32 s8, s7, 3
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX90A-NEXT: v_mov_b32_e32 v2, s8
; GFX90A-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB28_2:
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_mov_b64 s[8:9], exec
; GFX90A-NEXT: v_readfirstlane_b32 s10, v1
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
; GFX90A-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
+; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
; GFX90A-NEXT: s_cbranch_execz .LBB28_4
; GFX90A-NEXT: ; %bb.3:
; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX90A-NEXT: s_lshl_b32 s0, s3, 4
+; GFX90A-NEXT: s_lshl_b32 s0, s7, 4
; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX90A-NEXT: v_mov_b32_e32 v2, s0
; GFX90A-NEXT: ds_add_f32 v2, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB28_4:
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX90A-NEXT: v_add_f32_e32 v0, s10, v0
@@ -8555,14 +8547,14 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX90A-NEXT: ; implicit-def: $vgpr0
; GFX90A-NEXT: .LBB28_5: ; %ComputeLoop
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
-; GFX90A-NEXT: s_mov_b32 m0, s3
-; GFX90A-NEXT: v_readlane_b32 s8, v2, s3
-; GFX90A-NEXT: v_writelane_b32 v0, s6, m0
-; GFX90A-NEXT: s_lshl_b64 s[6:7], 1, s3
-; GFX90A-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]
-; GFX90A-NEXT: v_add_f32_e32 v1, s8, v1
+; GFX90A-NEXT: s_ff1_i32_b64 s2, s[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s3, v1
+; GFX90A-NEXT: s_mov_b32 m0, s2
+; GFX90A-NEXT: v_readlane_b32 s7, v2, s2
+; GFX90A-NEXT: v_writelane_b32 v0, s3, m0
+; GFX90A-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX90A-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX90A-NEXT: v_add_f32_e32 v1, s7, v1
; GFX90A-NEXT: s_cbranch_scc1 .LBB28_5
; GFX90A-NEXT: ; %bb.6: ; %ComputeEnd
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -8573,60 +8565,58 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX90A-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX90A-NEXT: s_cbranch_execz .LBB28_8
; GFX90A-NEXT: ; %bb.7:
-; GFX90A-NEXT: v_mov_b32_e32 v2, s2
+; GFX90A-NEXT: v_mov_b32_e32 v2, s6
; GFX90A-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: .LBB28_8:
; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX90A-NEXT: v_readfirstlane_b32 s2, v2
-; GFX90A-NEXT: v_add_f32_e32 v0, s2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v2, s2
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v2
+; GFX90A-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s0
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX90A-NEXT: global_store_dword v1, v0, s[4:5]
; GFX90A-NEXT: s_endpgm
;
; GFX908-LABEL: local_ds_fadd:
; GFX908: ; %bb.0:
-; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX908-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX908-NEXT: s_mov_b64 s[0:1], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_add_i32 s3, s3, 4
+; GFX908-NEXT: s_add_i32 s7, s7, 4
; GFX908-NEXT: ; implicit-def: $vgpr1
-; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT: s_cbranch_execz .LBB28_2
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX908-NEXT: s_lshl_b32 s8, s3, 3
+; GFX908-NEXT: s_lshl_b32 s8, s7, 3
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX908-NEXT: v_mov_b32_e32 v2, s8
; GFX908-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: .LBB28_2:
-; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: s_mov_b64 s[8:9], exec
; GFX908-NEXT: v_readfirstlane_b32 s10, v1
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
; GFX908-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GFX908-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
+; GFX908-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
; GFX908-NEXT: s_cbranch_execz .LBB28_4
; GFX908-NEXT: ; %bb.3:
; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX908-NEXT: s_lshl_b32 s0, s3, 4
+; GFX908-NEXT: s_lshl_b32 s0, s7, 4
; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX908-NEXT: v_mov_b32_e32 v2, s0
; GFX908-NEXT: ds_add_f32 v2, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: .LBB28_4:
-; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX908-NEXT: v_add_f32_e32 v0, s10, v0
@@ -8637,14 +8627,14 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX908-NEXT: ; implicit-def: $vgpr0
; GFX908-NEXT: .LBB28_5: ; %ComputeLoop
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX908-NEXT: v_readfirstlane_b32 s6, v1
-; GFX908-NEXT: s_mov_b32 m0, s3
-; GFX908-NEXT: v_readlane_b32 s8, v2, s3
-; GFX908-NEXT: v_writelane_b32 v0, s6, m0
-; GFX908-NEXT: s_lshl_b64 s[6:7], 1, s3
-; GFX908-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]
-; GFX908-NEXT: v_add_f32_e32 v1, s8, v1
+; GFX908-NEXT: s_ff1_i32_b64 s2, s[0:1]
+; GFX908-NEXT: v_readfirstlane_b32 s3, v1
+; GFX908-NEXT: s_mov_b32 m0, s2
+; GFX908-NEXT: v_readlane_b32 s7, v2, s2
+; GFX908-NEXT: v_writelane_b32 v0, s3, m0
+; GFX908-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX908-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX908-NEXT: v_add_f32_e32 v1, s7, v1
; GFX908-NEXT: s_cbranch_scc1 .LBB28_5
; GFX908-NEXT: ; %bb.6: ; %ComputeEnd
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -8655,61 +8645,59 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX908-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX908-NEXT: s_cbranch_execz .LBB28_8
; GFX908-NEXT: ; %bb.7:
-; GFX908-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-NEXT: v_mov_b32_e32 v2, s6
; GFX908-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: .LBB28_8:
; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX908-NEXT: v_readfirstlane_b32 s2, v2
-; GFX908-NEXT: v_add_f32_e32 v0, s2, v0
-; GFX908-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-NEXT: v_readfirstlane_b32 s0, v2
+; GFX908-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX908-NEXT: v_mov_b32_e32 v2, s0
; GFX908-NEXT: v_mov_b32_e32 v1, 0
; GFX908-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX908-NEXT: global_store_dword v1, v0, s[4:5]
; GFX908-NEXT: s_endpgm
;
; GFX8-LABEL: local_ds_fadd:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX8-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX8-NEXT: s_mov_b64 s[0:1], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_add_i32 s3, s3, 4
+; GFX8-NEXT: s_add_i32 s7, s7, 4
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX8-NEXT: s_cbranch_execz .LBB28_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX8-NEXT: s_lshl_b32 s8, s3, 3
+; GFX8-NEXT: s_lshl_b32 s8, s7, 3
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s8
; GFX8-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB28_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_mov_b64 s[8:9], exec
; GFX8-NEXT: v_readfirstlane_b32 s10, v1
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GFX8-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
+; GFX8-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
; GFX8-NEXT: s_cbranch_execz .LBB28_4
; GFX8-NEXT: ; %bb.3:
; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX8-NEXT: s_lshl_b32 s0, s3, 4
+; GFX8-NEXT: s_lshl_b32 s0, s7, 4
; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: ds_add_f32 v2, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB28_4:
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX8-NEXT: v_add_f32_e32 v0, s10, v0
@@ -8720,14 +8708,14 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: .LBB28_5: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX8-NEXT: v_readfirstlane_b32 s6, v1
-; GFX8-NEXT: s_mov_b32 m0, s3
-; GFX8-NEXT: v_readlane_b32 s8, v2, s3
-; GFX8-NEXT: v_writelane_b32 v0, s6, m0
-; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s3
-; GFX8-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]
-; GFX8-NEXT: v_add_f32_e32 v1, s8, v1
+; GFX8-NEXT: s_ff1_i32_b64 s2, s[0:1]
+; GFX8-NEXT: v_readfirstlane_b32 s3, v1
+; GFX8-NEXT: s_mov_b32 m0, s2
+; GFX8-NEXT: v_readlane_b32 s7, v2, s2
+; GFX8-NEXT: v_writelane_b32 v0, s3, m0
+; GFX8-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX8-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-NEXT: v_add_f32_e32 v1, s7, v1
; GFX8-NEXT: s_cbranch_scc1 .LBB28_5
; GFX8-NEXT: ; %bb.6: ; %ComputeEnd
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -8738,38 +8726,36 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX8-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX8-NEXT: s_cbranch_execz .LBB28_8
; GFX8-NEXT: ; %bb.7:
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: .LBB28_8:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX8-NEXT: v_readfirstlane_b32 s2, v2
-; GFX8-NEXT: v_add_f32_e32 v0, s2, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_store_dword v[0:1], v2
+; GFX8-NEXT: v_readfirstlane_b32 s0, v2
+; GFX8-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT: flat_store_dword v[3:4], v0
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: local_ds_fadd:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX7-NEXT: s_mov_b64 s[0:1], exec
; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_add_i32 s3, s3, 4
+; GFX7-NEXT: s_add_i32 s7, s7, 4
; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX7-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7-NEXT: s_cbranch_execz .LBB28_4
; GFX7-NEXT: ; %bb.1:
-; GFX7-NEXT: s_lshl_b32 s8, s3, 3
+; GFX7-NEXT: s_lshl_b32 s8, s7, 3
; GFX7-NEXT: v_mov_b32_e32 v2, s8
; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
@@ -8787,19 +8773,19 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB28_2
-; GFX7-NEXT: ; %bb.3: ; %Flow22
+; GFX7-NEXT: ; %bb.3: ; %Flow23
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: .LBB28_4: ; %Flow23
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT: .LBB28_4: ; %Flow24
+; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_mov_b64 s[8:9], exec
; GFX7-NEXT: v_readfirstlane_b32 s10, v1
; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0
; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1
; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
+; GFX7-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB28_7
; GFX7-NEXT: ; %bb.5:
-; GFX7-NEXT: s_lshl_b32 s0, s3, 4
+; GFX7-NEXT: s_lshl_b32 s0, s7, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s0
; GFX7-NEXT: ds_read_b32 v3, v1
; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
@@ -8817,8 +8803,8 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB28_6
-; GFX7-NEXT: .LBB28_7: ; %Flow21
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT: .LBB28_7: ; %Flow22
+; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX7-NEXT: v_add_f32_e32 v0, s10, v0
@@ -8829,16 +8815,16 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX7-NEXT: ; implicit-def: $vgpr0
; GFX7-NEXT: .LBB28_8: ; %ComputeLoop
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX7-NEXT: s_lshl_b64 s[6:7], 1, s3
-; GFX7-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]
-; GFX7-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
+; GFX7-NEXT: s_ff1_i32_b64 s7, s[0:1]
+; GFX7-NEXT: s_lshl_b64 s[2:3], 1, s7
+; GFX7-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX7-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GFX7-NEXT: v_readfirstlane_b32 s8, v1
-; GFX7-NEXT: v_readlane_b32 s9, v2, s3
-; GFX7-NEXT: s_mov_b32 m0, s3
+; GFX7-NEXT: v_readlane_b32 s9, v2, s7
+; GFX7-NEXT: s_mov_b32 m0, s7
; GFX7-NEXT: v_writelane_b32 v0, s8, m0
; GFX7-NEXT: v_add_f32_e32 v1, s9, v1
-; GFX7-NEXT: s_and_b64 vcc, exec, s[6:7]
+; GFX7-NEXT: s_and_b64 vcc, exec, s[2:3]
; GFX7-NEXT: s_cbranch_vccnz .LBB28_8
; GFX7-NEXT: ; %bb.9: ; %ComputeEnd
; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
@@ -8846,13 +8832,13 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7-NEXT: ; implicit-def: $vgpr2
; GFX7-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7-NEXT: s_xor_b64 s[6:7], exec, s[0:1]
+; GFX7-NEXT: s_xor_b64 s[2:3], exec, s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB28_13
; GFX7-NEXT: ; %bb.10:
-; GFX7-NEXT: v_mov_b32_e32 v3, s2
+; GFX7-NEXT: v_mov_b32_e32 v3, s6
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v3
-; GFX7-NEXT: s_mov_b64 s[2:3], 0
+; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB28_11: ; %atomicrmw.start8
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
@@ -8861,22 +8847,20 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v3, v4, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v4
-; GFX7-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7-NEXT: s_or_b64 s[6:7], s[0:1], s[6:7]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB28_11
; GFX7-NEXT: ; %bb.12: ; %Flow
-; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX7-NEXT: .LBB28_13: ; %Flow19
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX7-NEXT: v_readfirstlane_b32 s4, v2
-; GFX7-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: .LBB28_13: ; %Flow20
+; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7-NEXT: v_readfirstlane_b32 s0, v2
+; GFX7-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX6-LABEL: local_ds_fadd:
@@ -9017,34 +9001,33 @@ define amdgpu_kernel void @local_ds_fadd(ptr addrspace(1) %out, ptr addrspace(3)
define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrspace(3) %ptrf, i32 %idx) {
; GFX12-LABEL: local_ds_fadd_one_as:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX12-NEXT: s_mov_b32 s6, exec_lo
+; GFX12-NEXT: s_load_b128 s[4:7], s[4:5], 0x0
+; GFX12-NEXT: s_mov_b32 s2, exec_lo
; GFX12-NEXT: ; implicit-def: $vgpr1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX12-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX12-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_add_co_i32 s1, s3, 4
+; GFX12-NEXT: s_add_co_i32 s1, s7, 4
; GFX12-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX12-NEXT: s_cbranch_execz .LBB29_2
; GFX12-NEXT: ; %bb.1:
-; GFX12-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX12-NEXT: s_lshl_b32 s3, s1, 3
+; GFX12-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX12-NEXT: s_lshl_b32 s2, s1, 3
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1
; GFX12-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX12-NEXT: .LBB29_2:
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-NEXT: s_mov_b32 s7, exec_lo
; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s3, v1
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_readfirstlane_b32 s2, v1
; GFX12-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
-; GFX12-NEXT: s_mov_b32 s6, exec_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: s_mov_b32 s3, exec_lo
; GFX12-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX12-NEXT: s_cbranch_execz .LBB29_4
; GFX12-NEXT: ; %bb.3:
@@ -9057,28 +9040,28 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX12-NEXT: ds_add_f32 v2, v1
; GFX12-NEXT: .LBB29_4:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX12-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX12-NEXT: s_mov_b32 s1, exec_lo
; GFX12-NEXT: s_brev_b32 s0, 1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX12-NEXT: v_add_f32_e32 v0, s3, v0
+; GFX12-NEXT: v_add_f32_e32 v0, s2, v0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_cndmask_b32_e64 v1, v0, s3, vcc_lo
+; GFX12-NEXT: v_cndmask_b32_e64 v1, v0, s2, vcc_lo
; GFX12-NEXT: ; implicit-def: $vgpr0
; GFX12-NEXT: .LBB29_5: ; %ComputeLoop
; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_ctz_i32_b32 s3, s1
+; GFX12-NEXT: s_ctz_i32_b32 s2, s1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_readlane_b32 s6, v1, s3
-; GFX12-NEXT: v_writelane_b32 v0, s0, s3
-; GFX12-NEXT: s_lshl_b32 s3, 1, s3
+; GFX12-NEXT: v_readlane_b32 s3, v1, s2
+; GFX12-NEXT: v_writelane_b32 v0, s0, s2
+; GFX12-NEXT: s_lshl_b32 s2, 1, s2
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_and_not1_b32 s1, s1, s3
-; GFX12-NEXT: s_add_f32 s0, s0, s6
+; GFX12-NEXT: s_and_not1_b32 s1, s1, s2
+; GFX12-NEXT: s_add_f32 s0, s0, s3
; GFX12-NEXT: s_cbranch_scc1 .LBB29_5
; GFX12-NEXT: ; %bb.6: ; %ComputeEnd
; GFX12-NEXT: v_mbcnt_lo_u32_b32 v1, exec_lo, 0
@@ -9089,60 +9072,58 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_xor_b32 s1, exec_lo, s1
; GFX12-NEXT: ; %bb.7:
-; GFX12-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT: v_dual_mov_b32 v1, s6 :: v_dual_mov_b32 v2, s0
; GFX12-NEXT: ds_add_rtn_f32 v1, v1, v2
; GFX12-NEXT: ; %bb.8:
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_or_b32 exec_lo, exec_lo, s1
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-NEXT: s_wait_dscnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s2, v1
+; GFX12-NEXT: v_readfirstlane_b32 s0, v1
; GFX12-NEXT: s_wait_alu depctr_va_sdst(0)
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0
-; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s0, v0
+; GFX12-NEXT: v_cndmask_b32_e64 v0, v0, s0, vcc_lo
+; GFX12-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX12-NEXT: s_endpgm
;
; GFX942-LABEL: local_ds_fadd_one_as:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX942-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX942-NEXT: s_mov_b64 s[0:1], exec
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX942-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_add_i32 s3, s3, 4
+; GFX942-NEXT: s_add_i32 s7, s7, 4
; GFX942-NEXT: ; implicit-def: $vgpr1
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX942-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX942-NEXT: s_cbranch_execz .LBB29_2
; GFX942-NEXT: ; %bb.1:
; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX942-NEXT: s_lshl_b32 s8, s3, 3
+; GFX942-NEXT: s_lshl_b32 s8, s7, 3
; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX942-NEXT: v_mov_b32_e32 v2, s8
; GFX942-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX942-NEXT: .LBB29_2:
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: s_mov_b64 s[8:9], exec
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_readfirstlane_b32 s10, v1
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
; GFX942-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
; GFX942-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GFX942-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
+; GFX942-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
; GFX942-NEXT: s_cbranch_execz .LBB29_4
; GFX942-NEXT: ; %bb.3:
; GFX942-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX942-NEXT: s_lshl_b32 s0, s3, 4
+; GFX942-NEXT: s_lshl_b32 s0, s7, 4
; GFX942-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX942-NEXT: v_mov_b32_e32 v2, s0
; GFX942-NEXT: ds_add_f32 v2, v1
; GFX942-NEXT: .LBB29_4:
-; GFX942-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX942-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX942-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX942-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX942-NEXT: v_add_f32_e32 v0, s10, v0
@@ -9153,14 +9134,14 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX942-NEXT: ; implicit-def: $vgpr0
; GFX942-NEXT: .LBB29_5: ; %ComputeLoop
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX942-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX942-NEXT: v_readfirstlane_b32 s6, v1
-; GFX942-NEXT: s_mov_b32 m0, s3
-; GFX942-NEXT: v_readlane_b32 s8, v2, s3
-; GFX942-NEXT: v_writelane_b32 v0, s6, m0
-; GFX942-NEXT: s_lshl_b64 s[6:7], 1, s3
-; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]
-; GFX942-NEXT: v_add_f32_e32 v1, s8, v1
+; GFX942-NEXT: s_ff1_i32_b64 s2, s[0:1]
+; GFX942-NEXT: v_readfirstlane_b32 s3, v1
+; GFX942-NEXT: s_mov_b32 m0, s2
+; GFX942-NEXT: v_readlane_b32 s7, v2, s2
+; GFX942-NEXT: v_writelane_b32 v0, s3, m0
+; GFX942-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX942-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX942-NEXT: v_add_f32_e32 v1, s7, v1
; GFX942-NEXT: s_cbranch_scc1 .LBB29_5
; GFX942-NEXT: ; %bb.6: ; %ComputeEnd
; GFX942-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -9170,49 +9151,48 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX942-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX942-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX942-NEXT: ; %bb.7:
-; GFX942-NEXT: v_mov_b32_e32 v2, s2
+; GFX942-NEXT: v_mov_b32_e32 v2, s6
; GFX942-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX942-NEXT: ; %bb.8:
; GFX942-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_readfirstlane_b32 s2, v2
+; GFX942-NEXT: v_readfirstlane_b32 s0, v2
; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: s_nop 0
-; GFX942-NEXT: v_add_f32_e32 v0, s2, v0
-; GFX942-NEXT: v_mov_b32_e32 v2, s2
+; GFX942-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX942-NEXT: v_mov_b32_e32 v2, s0
; GFX942-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX942-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX942-NEXT: global_store_dword v1, v0, s[4:5]
; GFX942-NEXT: s_endpgm
;
; GFX11-LABEL: local_ds_fadd_one_as:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x8
-; GFX11-NEXT: s_mov_b32 s6, exec_lo
+; GFX11-NEXT: s_load_b128 s[4:7], s[4:5], 0x0
+; GFX11-NEXT: s_mov_b32 s2, exec_lo
; GFX11-NEXT: ; implicit-def: $vgpr1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX11-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX11-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_add_i32 s1, s3, 4
+; GFX11-NEXT: s_add_i32 s1, s7, 4
; GFX11-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-NEXT: s_cbranch_execz .LBB29_2
; GFX11-NEXT: ; %bb.1:
-; GFX11-NEXT: s_bcnt1_i32_b32 s3, s6
+; GFX11-NEXT: s_bcnt1_i32_b32 s2, s2
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX11-NEXT: s_lshl_b32 s3, s1, 3
+; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX11-NEXT: s_lshl_b32 s2, s1, 3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mul_f32 v1, 0x42280000, v1
+; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mul_f32 v1, 0x42280000, v1
; GFX11-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX11-NEXT: .LBB29_2:
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-NEXT: s_mov_b32 s7, exec_lo
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_readfirstlane_b32 s3, v1
+; GFX11-NEXT: v_readfirstlane_b32 s2, v1
; GFX11-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
-; GFX11-NEXT: s_mov_b32 s6, exec_lo
+; GFX11-NEXT: s_mov_b32 s3, exec_lo
; GFX11-NEXT: v_cmpx_eq_u32_e32 0, v2
; GFX11-NEXT: s_cbranch_execz .LBB29_4
; GFX11-NEXT: ; %bb.3:
@@ -9224,25 +9204,25 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mul_f32 v1, 0x42280000, v1
; GFX11-NEXT: ds_add_f32 v2, v1
; GFX11-NEXT: .LBB29_4:
-; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX11-NEXT: v_bfrev_b32_e32 v1, 1
; GFX11-NEXT: s_mov_b32 s0, exec_lo
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX11-NEXT: v_add_f32_e32 v0, s3, v0
+; GFX11-NEXT: v_add_f32_e32 v0, s2, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, s3, vcc_lo
+; GFX11-NEXT: v_cndmask_b32_e64 v2, v0, s2, vcc_lo
; GFX11-NEXT: ; implicit-def: $vgpr0
; GFX11-NEXT: .LBB29_5: ; %ComputeLoop
; GFX11-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-NEXT: s_ctz_i32_b32 s1, s0
-; GFX11-NEXT: v_readfirstlane_b32 s3, v1
+; GFX11-NEXT: v_readfirstlane_b32 s2, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_readlane_b32 s6, v2, s1
-; GFX11-NEXT: v_writelane_b32 v0, s3, s1
+; GFX11-NEXT: v_readlane_b32 s3, v2, s1
+; GFX11-NEXT: v_writelane_b32 v0, s2, s1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_add_f32_e32 v1, s6, v1
+; GFX11-NEXT: v_add_f32_e32 v1, s3, v1
; GFX11-NEXT: s_lshl_b32 s1, 1, s1
; GFX11-NEXT: s_and_not1_b32 s0, s0, s1
; GFX11-NEXT: s_cbranch_scc1 .LBB29_5
@@ -9254,35 +9234,34 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX11-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX11-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX11-NEXT: ; %bb.7:
-; GFX11-NEXT: v_mov_b32_e32 v2, s2
+; GFX11-NEXT: v_mov_b32_e32 v2, s6
; GFX11-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX11-NEXT: ; %bb.8:
; GFX11-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s0, v2
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s2, v0
-; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo
-; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_f32 v0, s0, v0
+; GFX11-NEXT: v_cndmask_b32_e64 v0, v0, s0, vcc_lo
+; GFX11-NEXT: global_store_b32 v1, v0, s[4:5]
; GFX11-NEXT: s_endpgm
;
; GFX10-LABEL: local_ds_fadd_one_as:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
-; GFX10-NEXT: s_mov_b32 s6, exec_lo
+; GFX10-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
+; GFX10-NEXT: s_mov_b32 s2, exec_lo
; GFX10-NEXT: ; implicit-def: $vgpr1
-; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s6, 0
+; GFX10-NEXT: v_mbcnt_lo_u32_b32 v0, s2, 0
; GFX10-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_add_i32 s1, s3, 4
+; GFX10-NEXT: s_add_i32 s1, s7, 4
; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10-NEXT: s_cbranch_execz .LBB29_2
; GFX10-NEXT: ; %bb.1:
-; GFX10-NEXT: s_bcnt1_i32_b32 s3, s6
-; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s3
-; GFX10-NEXT: s_lshl_b32 s3, s1, 3
-; GFX10-NEXT: v_mov_b32_e32 v2, s3
+; GFX10-NEXT: s_bcnt1_i32_b32 s2, s2
+; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v1, s2
+; GFX10-NEXT: s_lshl_b32 s2, s1, 3
+; GFX10-NEXT: v_mov_b32_e32 v2, s2
; GFX10-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX10-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX10-NEXT: .LBB29_2:
@@ -9290,10 +9269,10 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX10-NEXT: s_mov_b32 s7, exec_lo
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_readfirstlane_b32 s3, v1
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
; GFX10-NEXT: v_mbcnt_lo_u32_b32 v2, s7, 0
; GFX10-NEXT: v_cmp_eq_u32_e64 s0, 0, v2
-; GFX10-NEXT: s_and_saveexec_b32 s6, s0
+; GFX10-NEXT: s_and_saveexec_b32 s3, s0
; GFX10-NEXT: s_cbranch_execz .LBB29_4
; GFX10-NEXT: ; %bb.3:
; GFX10-NEXT: s_bcnt1_i32_b32 s0, s7
@@ -9304,21 +9283,21 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX10-NEXT: ds_add_f32 v2, v1
; GFX10-NEXT: .LBB29_4:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s6
+; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s3
; GFX10-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX10-NEXT: v_bfrev_b32_e32 v1, 1
; GFX10-NEXT: s_mov_b32 s0, exec_lo
; GFX10-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
-; GFX10-NEXT: v_add_f32_e32 v0, s3, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, s3, vcc_lo
+; GFX10-NEXT: v_add_f32_e32 v0, s2, v0
+; GFX10-NEXT: v_cndmask_b32_e64 v2, v0, s2, vcc_lo
; GFX10-NEXT: ; implicit-def: $vgpr0
; GFX10-NEXT: .LBB29_5: ; %ComputeLoop
; GFX10-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX10-NEXT: s_ff1_i32_b32 s1, s0
-; GFX10-NEXT: v_readfirstlane_b32 s3, v1
-; GFX10-NEXT: v_readlane_b32 s6, v2, s1
-; GFX10-NEXT: v_writelane_b32 v0, s3, s1
-; GFX10-NEXT: v_add_f32_e32 v1, s6, v1
+; GFX10-NEXT: v_readfirstlane_b32 s2, v1
+; GFX10-NEXT: v_readlane_b32 s3, v2, s1
+; GFX10-NEXT: v_writelane_b32 v0, s2, s1
+; GFX10-NEXT: v_add_f32_e32 v1, s3, v1
; GFX10-NEXT: s_lshl_b32 s1, 1, s1
; GFX10-NEXT: s_andn2_b32 s0, s0, s1
; GFX10-NEXT: s_cbranch_scc1 .LBB29_5
@@ -9329,58 +9308,57 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX10-NEXT: s_and_saveexec_b32 s0, vcc_lo
; GFX10-NEXT: s_xor_b32 s0, exec_lo, s0
; GFX10-NEXT: ; %bb.7:
-; GFX10-NEXT: v_mov_b32_e32 v2, s2
+; GFX10-NEXT: v_mov_b32_e32 v2, s6
; GFX10-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX10-NEXT: ; %bb.8:
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
; GFX10-NEXT: s_or_b32 exec_lo, exec_lo, s0
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_readfirstlane_b32 s2, v2
+; GFX10-NEXT: v_readfirstlane_b32 s0, v2
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_add_f32_e32 v0, s2, v0
-; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s2, vcc_lo
-; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX10-NEXT: v_cndmask_b32_e64 v0, v0, s0, vcc_lo
+; GFX10-NEXT: global_store_dword v1, v0, s[4:5]
; GFX10-NEXT: s_endpgm
;
; GFX90A-LABEL: local_ds_fadd_one_as:
; GFX90A: ; %bb.0:
-; GFX90A-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX90A-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX90A-NEXT: s_mov_b64 s[0:1], exec
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX90A-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_add_i32 s3, s3, 4
+; GFX90A-NEXT: s_add_i32 s7, s7, 4
; GFX90A-NEXT: ; implicit-def: $vgpr1
-; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX90A-NEXT: s_cbranch_execz .LBB29_2
; GFX90A-NEXT: ; %bb.1:
; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX90A-NEXT: s_lshl_b32 s8, s3, 3
+; GFX90A-NEXT: s_lshl_b32 s8, s7, 3
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX90A-NEXT: v_mov_b32_e32 v2, s8
; GFX90A-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX90A-NEXT: .LBB29_2:
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: s_mov_b64 s[8:9], exec
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: v_readfirstlane_b32 s10, v1
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
; GFX90A-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
; GFX90A-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GFX90A-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
+; GFX90A-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
; GFX90A-NEXT: s_cbranch_execz .LBB29_4
; GFX90A-NEXT: ; %bb.3:
; GFX90A-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX90A-NEXT: s_lshl_b32 s0, s3, 4
+; GFX90A-NEXT: s_lshl_b32 s0, s7, 4
; GFX90A-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX90A-NEXT: v_mov_b32_e32 v2, s0
; GFX90A-NEXT: ds_add_f32 v2, v1
; GFX90A-NEXT: .LBB29_4:
-; GFX90A-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX90A-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX90A-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX90A-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX90A-NEXT: v_add_f32_e32 v0, s10, v0
@@ -9391,14 +9369,14 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX90A-NEXT: ; implicit-def: $vgpr0
; GFX90A-NEXT: .LBB29_5: ; %ComputeLoop
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX90A-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX90A-NEXT: v_readfirstlane_b32 s6, v1
-; GFX90A-NEXT: s_mov_b32 m0, s3
-; GFX90A-NEXT: v_readlane_b32 s8, v2, s3
-; GFX90A-NEXT: v_writelane_b32 v0, s6, m0
-; GFX90A-NEXT: s_lshl_b64 s[6:7], 1, s3
-; GFX90A-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]
-; GFX90A-NEXT: v_add_f32_e32 v1, s8, v1
+; GFX90A-NEXT: s_ff1_i32_b64 s2, s[0:1]
+; GFX90A-NEXT: v_readfirstlane_b32 s3, v1
+; GFX90A-NEXT: s_mov_b32 m0, s2
+; GFX90A-NEXT: v_readlane_b32 s7, v2, s2
+; GFX90A-NEXT: v_writelane_b32 v0, s3, m0
+; GFX90A-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX90A-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX90A-NEXT: v_add_f32_e32 v1, s7, v1
; GFX90A-NEXT: s_cbranch_scc1 .LBB29_5
; GFX90A-NEXT: ; %bb.6: ; %ComputeEnd
; GFX90A-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -9408,58 +9386,57 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX90A-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX90A-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX90A-NEXT: ; %bb.7:
-; GFX90A-NEXT: v_mov_b32_e32 v2, s2
+; GFX90A-NEXT: v_mov_b32_e32 v2, s6
; GFX90A-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX90A-NEXT: ; %bb.8:
; GFX90A-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_readfirstlane_b32 s2, v2
-; GFX90A-NEXT: v_add_f32_e32 v0, s2, v0
-; GFX90A-NEXT: v_mov_b32_e32 v2, s2
+; GFX90A-NEXT: v_readfirstlane_b32 s0, v2
+; GFX90A-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX90A-NEXT: v_mov_b32_e32 v2, s0
; GFX90A-NEXT: v_mov_b32_e32 v1, 0
; GFX90A-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX90A-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX90A-NEXT: global_store_dword v1, v0, s[4:5]
; GFX90A-NEXT: s_endpgm
;
; GFX908-LABEL: local_ds_fadd_one_as:
; GFX908: ; %bb.0:
-; GFX908-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX908-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX908-NEXT: s_mov_b64 s[0:1], exec
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX908-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_add_i32 s3, s3, 4
+; GFX908-NEXT: s_add_i32 s7, s7, 4
; GFX908-NEXT: ; implicit-def: $vgpr1
-; GFX908-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX908-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX908-NEXT: s_cbranch_execz .LBB29_2
; GFX908-NEXT: ; %bb.1:
; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX908-NEXT: s_lshl_b32 s8, s3, 3
+; GFX908-NEXT: s_lshl_b32 s8, s7, 3
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX908-NEXT: v_mov_b32_e32 v2, s8
; GFX908-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX908-NEXT: .LBB29_2:
-; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: s_mov_b64 s[8:9], exec
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_readfirstlane_b32 s10, v1
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
; GFX908-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
; GFX908-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GFX908-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
+; GFX908-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
; GFX908-NEXT: s_cbranch_execz .LBB29_4
; GFX908-NEXT: ; %bb.3:
; GFX908-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX908-NEXT: s_lshl_b32 s0, s3, 4
+; GFX908-NEXT: s_lshl_b32 s0, s7, 4
; GFX908-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX908-NEXT: v_mov_b32_e32 v2, s0
; GFX908-NEXT: ds_add_f32 v2, v1
; GFX908-NEXT: .LBB29_4:
-; GFX908-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX908-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX908-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX908-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX908-NEXT: v_add_f32_e32 v0, s10, v0
@@ -9470,14 +9447,14 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX908-NEXT: ; implicit-def: $vgpr0
; GFX908-NEXT: .LBB29_5: ; %ComputeLoop
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX908-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX908-NEXT: v_readfirstlane_b32 s6, v1
-; GFX908-NEXT: s_mov_b32 m0, s3
-; GFX908-NEXT: v_readlane_b32 s8, v2, s3
-; GFX908-NEXT: v_writelane_b32 v0, s6, m0
-; GFX908-NEXT: s_lshl_b64 s[6:7], 1, s3
-; GFX908-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]
-; GFX908-NEXT: v_add_f32_e32 v1, s8, v1
+; GFX908-NEXT: s_ff1_i32_b64 s2, s[0:1]
+; GFX908-NEXT: v_readfirstlane_b32 s3, v1
+; GFX908-NEXT: s_mov_b32 m0, s2
+; GFX908-NEXT: v_readlane_b32 s7, v2, s2
+; GFX908-NEXT: v_writelane_b32 v0, s3, m0
+; GFX908-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX908-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX908-NEXT: v_add_f32_e32 v1, s7, v1
; GFX908-NEXT: s_cbranch_scc1 .LBB29_5
; GFX908-NEXT: ; %bb.6: ; %ComputeEnd
; GFX908-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -9487,59 +9464,58 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX908-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX908-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX908-NEXT: ; %bb.7:
-; GFX908-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-NEXT: v_mov_b32_e32 v2, s6
; GFX908-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX908-NEXT: ; %bb.8:
; GFX908-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_readfirstlane_b32 s2, v2
-; GFX908-NEXT: v_add_f32_e32 v0, s2, v0
-; GFX908-NEXT: v_mov_b32_e32 v2, s2
+; GFX908-NEXT: v_readfirstlane_b32 s0, v2
+; GFX908-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX908-NEXT: v_mov_b32_e32 v2, s0
; GFX908-NEXT: v_mov_b32_e32 v1, 0
; GFX908-NEXT: v_cndmask_b32_e32 v0, v0, v2, vcc
-; GFX908-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX908-NEXT: global_store_dword v1, v0, s[4:5]
; GFX908-NEXT: s_endpgm
;
; GFX8-LABEL: local_ds_fadd_one_as:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x8
+; GFX8-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX8-NEXT: s_mov_b64 s[0:1], exec
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v0, s0, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v0, s1, v0
; GFX8-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_add_i32 s3, s3, 4
+; GFX8-NEXT: s_add_i32 s7, s7, 4
; GFX8-NEXT: ; implicit-def: $vgpr1
; GFX8-NEXT: s_mov_b32 m0, -1
-; GFX8-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX8-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX8-NEXT: s_cbranch_execz .LBB29_2
; GFX8-NEXT: ; %bb.1:
; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
-; GFX8-NEXT: s_lshl_b32 s8, s3, 3
+; GFX8-NEXT: s_lshl_b32 s8, s7, 3
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s8
; GFX8-NEXT: ds_add_rtn_f32 v1, v2, v1
; GFX8-NEXT: .LBB29_2:
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: s_mov_b64 s[8:9], exec
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s10, v1
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v1, s8, 0
; GFX8-NEXT: v_mbcnt_hi_u32_b32 v1, s9, v1
; GFX8-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GFX8-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
+; GFX8-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
; GFX8-NEXT: s_cbranch_execz .LBB29_4
; GFX8-NEXT: ; %bb.3:
; GFX8-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v1, s0
-; GFX8-NEXT: s_lshl_b32 s0, s3, 4
+; GFX8-NEXT: s_lshl_b32 s0, s7, 4
; GFX8-NEXT: v_mul_f32_e32 v1, 0x42280000, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: ds_add_f32 v2, v1
; GFX8-NEXT: .LBB29_4:
-; GFX8-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX8-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX8-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX8-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX8-NEXT: v_add_f32_e32 v0, s10, v0
@@ -9550,14 +9526,14 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX8-NEXT: ; implicit-def: $vgpr0
; GFX8-NEXT: .LBB29_5: ; %ComputeLoop
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX8-NEXT: v_readfirstlane_b32 s6, v1
-; GFX8-NEXT: s_mov_b32 m0, s3
-; GFX8-NEXT: v_readlane_b32 s8, v2, s3
-; GFX8-NEXT: v_writelane_b32 v0, s6, m0
-; GFX8-NEXT: s_lshl_b64 s[6:7], 1, s3
-; GFX8-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]
-; GFX8-NEXT: v_add_f32_e32 v1, s8, v1
+; GFX8-NEXT: s_ff1_i32_b64 s2, s[0:1]
+; GFX8-NEXT: v_readfirstlane_b32 s3, v1
+; GFX8-NEXT: s_mov_b32 m0, s2
+; GFX8-NEXT: v_readlane_b32 s7, v2, s2
+; GFX8-NEXT: v_writelane_b32 v0, s3, m0
+; GFX8-NEXT: s_lshl_b64 s[2:3], 1, s2
+; GFX8-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX8-NEXT: v_add_f32_e32 v1, s7, v1
; GFX8-NEXT: s_cbranch_scc1 .LBB29_5
; GFX8-NEXT: ; %bb.6: ; %ComputeEnd
; GFX8-NEXT: v_mbcnt_lo_u32_b32 v2, exec_lo, 0
@@ -9567,37 +9543,36 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_xor_b64 s[0:1], exec, s[0:1]
; GFX8-NEXT: ; %bb.7:
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s6
; GFX8-NEXT: s_mov_b32 m0, -1
; GFX8-NEXT: ds_add_rtn_f32 v2, v2, v1
; GFX8-NEXT: ; %bb.8:
; GFX8-NEXT: s_or_b64 exec, exec, s[0:1]
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_readfirstlane_b32 s2, v2
-; GFX8-NEXT: v_add_f32_e32 v0, s2, v0
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
-; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: flat_store_dword v[0:1], v2
+; GFX8-NEXT: v_readfirstlane_b32 s0, v2
+; GFX8-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX8-NEXT: v_mov_b32_e32 v1, s0
+; GFX8-NEXT: v_mov_b32_e32 v3, s4
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
+; GFX8-NEXT: flat_store_dword v[3:4], v0
; GFX8-NEXT: s_endpgm
;
; GFX7-LABEL: local_ds_fadd_one_as:
; GFX7: ; %bb.0:
-; GFX7-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x2
+; GFX7-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x0
; GFX7-NEXT: s_mov_b64 s[0:1], exec
; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v0, s0, 0
; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v0, s1, v0
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: s_add_i32 s3, s3, 4
+; GFX7-NEXT: s_add_i32 s7, s7, 4
; GFX7-NEXT: ; implicit-def: $vgpr1
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; GFX7-NEXT: s_and_saveexec_b64 s[2:3], vcc
; GFX7-NEXT: s_cbranch_execz .LBB29_4
; GFX7-NEXT: ; %bb.1:
-; GFX7-NEXT: s_lshl_b32 s8, s3, 3
+; GFX7-NEXT: s_lshl_b32 s8, s7, 3
; GFX7-NEXT: v_mov_b32_e32 v2, s8
; GFX7-NEXT: ds_read_b32 v1, v2
; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[0:1]
@@ -9615,19 +9590,19 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: s_or_b64 s[8:9], s[0:1], s[8:9]
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB29_2
-; GFX7-NEXT: ; %bb.3: ; %Flow22
+; GFX7-NEXT: ; %bb.3: ; %Flow23
; GFX7-NEXT: s_or_b64 exec, exec, s[8:9]
-; GFX7-NEXT: .LBB29_4: ; %Flow23
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT: .LBB29_4: ; %Flow24
+; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: s_mov_b64 s[8:9], exec
; GFX7-NEXT: v_readfirstlane_b32 s10, v1
; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v1, s8, 0
; GFX7-NEXT: v_mbcnt_hi_u32_b32_e32 v1, s9, v1
; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], 0, v1
-; GFX7-NEXT: s_and_saveexec_b64 s[6:7], s[0:1]
+; GFX7-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB29_7
; GFX7-NEXT: ; %bb.5:
-; GFX7-NEXT: s_lshl_b32 s0, s3, 4
+; GFX7-NEXT: s_lshl_b32 s0, s7, 4
; GFX7-NEXT: v_mov_b32_e32 v1, s0
; GFX7-NEXT: ds_read_b32 v3, v1
; GFX7-NEXT: s_bcnt1_i32_b64 s0, s[8:9]
@@ -9645,8 +9620,8 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: v_mov_b32_e32 v3, v4
; GFX7-NEXT: s_andn2_b64 exec, exec, s[8:9]
; GFX7-NEXT: s_cbranch_execnz .LBB29_6
-; GFX7-NEXT: .LBB29_7: ; %Flow21
-; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
+; GFX7-NEXT: .LBB29_7: ; %Flow22
+; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
; GFX7-NEXT: v_cvt_f32_ubyte0_e32 v0, v0
; GFX7-NEXT: v_mul_f32_e32 v0, 0x42280000, v0
; GFX7-NEXT: v_add_f32_e32 v0, s10, v0
@@ -9657,16 +9632,16 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: ; implicit-def: $vgpr0
; GFX7-NEXT: .LBB29_8: ; %ComputeLoop
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX7-NEXT: s_ff1_i32_b64 s3, s[0:1]
-; GFX7-NEXT: s_lshl_b64 s[6:7], 1, s3
-; GFX7-NEXT: s_andn2_b64 s[0:1], s[0:1], s[6:7]
-; GFX7-NEXT: v_cmp_ne_u64_e64 s[6:7], s[0:1], 0
+; GFX7-NEXT: s_ff1_i32_b64 s7, s[0:1]
+; GFX7-NEXT: s_lshl_b64 s[2:3], 1, s7
+; GFX7-NEXT: s_andn2_b64 s[0:1], s[0:1], s[2:3]
+; GFX7-NEXT: v_cmp_ne_u64_e64 s[2:3], s[0:1], 0
; GFX7-NEXT: v_readfirstlane_b32 s8, v1
-; GFX7-NEXT: v_readlane_b32 s9, v2, s3
-; GFX7-NEXT: s_mov_b32 m0, s3
+; GFX7-NEXT: v_readlane_b32 s9, v2, s7
+; GFX7-NEXT: s_mov_b32 m0, s7
; GFX7-NEXT: v_writelane_b32 v0, s8, m0
; GFX7-NEXT: v_add_f32_e32 v1, s9, v1
-; GFX7-NEXT: s_and_b64 vcc, exec, s[6:7]
+; GFX7-NEXT: s_and_b64 vcc, exec, s[2:3]
; GFX7-NEXT: s_cbranch_vccnz .LBB29_8
; GFX7-NEXT: ; %bb.9: ; %ComputeEnd
; GFX7-NEXT: v_mbcnt_lo_u32_b32_e64 v2, exec_lo, 0
@@ -9674,13 +9649,13 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; GFX7-NEXT: ; implicit-def: $vgpr2
; GFX7-NEXT: s_and_saveexec_b64 s[0:1], vcc
-; GFX7-NEXT: s_xor_b64 s[6:7], exec, s[0:1]
+; GFX7-NEXT: s_xor_b64 s[2:3], exec, s[0:1]
; GFX7-NEXT: s_cbranch_execz .LBB29_13
; GFX7-NEXT: ; %bb.10:
-; GFX7-NEXT: v_mov_b32_e32 v3, s2
+; GFX7-NEXT: v_mov_b32_e32 v3, s6
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_b32 v2, v3
-; GFX7-NEXT: s_mov_b64 s[2:3], 0
+; GFX7-NEXT: s_mov_b64 s[6:7], 0
; GFX7-NEXT: .LBB29_11: ; %atomicrmw.start8
; GFX7-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
@@ -9689,22 +9664,20 @@ define amdgpu_kernel void @local_ds_fadd_one_as(ptr addrspace(1) %out, ptr addrs
; GFX7-NEXT: ds_cmpst_rtn_b32 v2, v3, v4, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_cmp_eq_u32_e64 s[0:1], v2, v4
-; GFX7-NEXT: s_or_b64 s[2:3], s[0:1], s[2:3]
-; GFX7-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX7-NEXT: s_or_b64 s[6:7], s[0:1], s[6:7]
+; GFX7-NEXT: s_andn2_b64 exec, exec, s[6:7]
; GFX7-NEXT: s_cbranch_execnz .LBB29_11
; GFX7-NEXT: ; %bb.12: ; %Flow
-; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
-; GFX7-NEXT: .LBB29_13: ; %Flow19
; GFX7-NEXT: s_or_b64 exec, exec, s[6:7]
-; GFX7-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX7-NEXT: v_readfirstlane_b32 s4, v2
-; GFX7-NEXT: v_add_f32_e32 v0, s4, v0
-; GFX7-NEXT: v_mov_b32_e32 v1, s4
-; GFX7-NEXT: s_mov_b32 s3, 0xf000
-; GFX7-NEXT: s_mov_b32 s2, -1
+; GFX7-NEXT: .LBB29_13: ; %Flow20
+; GFX7-NEXT: s_or_b64 exec, exec, s[2:3]
+; GFX7-NEXT: v_readfirstlane_b32 s0, v2
+; GFX7-NEXT: v_add_f32_e32 v0, s0, v0
+; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: s_mov_b32 s7, 0xf000
+; GFX7-NEXT: s_mov_b32 s6, -1
; GFX7-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX7-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX7-NEXT: s_endpgm
;
; GFX6-LABEL: local_ds_fadd_one_as:
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomics.ll b/llvm/test/CodeGen/AMDGPU/local-atomics.ll
index 7461122cdebd3..b2c66dda5453d 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomics.ll
@@ -10,9 +10,12 @@
; SICIVI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
-; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],
+; SI-DAG: s_load_dword s{{[0-9]+}}, s[4:5], 0xb
+; SI-DAG: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x9
+; CIVI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x9|0x24}}
+; GFX9-DAG: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
-; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], s{{[0-9]+}}
; GCN: ds_wrxchg_rtn_b32 [[RESULT:v[0-9]+]], [[VPTR]], [[DATA]]
; GCN: buffer_store_dword [[RESULT]],
; GCN: s_endpgm
@@ -57,9 +60,12 @@ define amdgpu_kernel void @lds_atomic_xchg_ret_f32_offset(ptr addrspace(1) %out,
; SICIVI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
-; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],
+; SI-DAG: s_load_dword s{{[0-9]+}}, s[4:5], 0xb
+; SI-DAG: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x9
+; CIVI-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x9|0x24}}
+; GFX9-DAG: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
-; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], s{{[0-9]+}}
; GCN: ds_add_rtn_u32 [[RESULT:v[0-9]+]], [[VPTR]], [[DATA]]
; GCN: buffer_store_dword [[RESULT]],
; GCN: s_endpgm
@@ -410,9 +416,9 @@ define amdgpu_kernel void @lds_atomic_umax_ret_i32_offset(ptr addrspace(1) %out,
; SICIVI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
-; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],
+; GCN-DAG: s_load_dword s{{[0-9]+}}, s[4:5], {{0x9|0x24}}
; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
-; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], s{{[0-9]+}}
; GCN: ds_wrxchg_rtn_b32 [[RESULT:v[0-9]+]], [[VPTR]], [[DATA]]
; GCN: s_endpgm
define amdgpu_kernel void @lds_atomic_xchg_noret_i32(ptr addrspace(3) %ptr) nounwind {
@@ -436,9 +442,9 @@ define amdgpu_kernel void @lds_atomic_xchg_noret_i32_offset(ptr addrspace(3) %pt
; SICIVI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
-; GCN-DAG: s_load_dword [[SPTR:s[0-9]+]],
+; GCN-DAG: s_load_dword s{{[0-9]+}}, s[4:5], {{0x9|0x24}}
; GCN-DAG: v_mov_b32_e32 [[DATA:v[0-9]+]], 4
-; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[SPTR]]
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], s{{[0-9]+}}
; GCN: ds_add_u32 [[VPTR]], [[DATA]]
; GCN: s_endpgm
define amdgpu_kernel void @lds_atomic_add_noret_i32(ptr addrspace(3) %ptr) nounwind {
diff --git a/llvm/test/CodeGen/AMDGPU/local-atomics64.ll b/llvm/test/CodeGen/AMDGPU/local-atomics64.ll
index 3bf2a4e56a87d..91a760323bc37 100644
--- a/llvm/test/CodeGen/AMDGPU/local-atomics64.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-atomics64.ll
@@ -69,11 +69,12 @@ define amdgpu_kernel void @lds_atomic_add_ret_i64(ptr addrspace(1) %out, ptr add
; SICIVI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
-; SI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
-; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
+; SI-DAG: s_load_dword s{{[0-9]+}}, s[4:5], 0xb
+; SI-DAG: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x9
+; GFX89-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x24
; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 9
; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0
-; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], s{{[0-9]+}}
; GCN: ds_add_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v[[[LOVDATA]]:[[HIVDATA]]] offset:32
; GCN: buffer_store_dwordx2 [[RESULT]],
; GCN: s_endpgm
@@ -656,11 +657,12 @@ define amdgpu_kernel void @lds_atomic_inc_ret_i64(ptr addrspace(1) %out, ptr add
; SICIVI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
-; SI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
-; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
+; SI-DAG: s_load_dword s{{[0-9]+}}, s[4:5], 0xb
+; SI-DAG: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x9
+; GFX89-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x24
; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 9
; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0
-; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], s{{[0-9]+}}
; GCN: ds_inc_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v[[[LOVDATA]]:[[HIVDATA]]] offset:32
; GCN: buffer_store_dwordx2 [[RESULT]],
; GCN: s_endpgm
@@ -715,11 +717,12 @@ define amdgpu_kernel void @lds_atomic_dec_ret_i64(ptr addrspace(1) %out, ptr add
; SICIVI-DAG: s_mov_b32 m0
; GFX9-NOT: m0
-; SI-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0xb
-; GFX89-DAG: s_load_dword [[PTR:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
+; SI-DAG: s_load_dword s{{[0-9]+}}, s[4:5], 0xb
+; SI-DAG: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x9
+; GFX89-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, 0x24
; GCN-DAG: v_mov_b32_e32 v[[LOVDATA:[0-9]+]], 9
; GCN-DAG: v_mov_b32_e32 v[[HIVDATA:[0-9]+]], 0
-; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], [[PTR]]
+; GCN-DAG: v_mov_b32_e32 [[VPTR:v[0-9]+]], s{{[0-9]+}}
; GCN: ds_dec_rtn_u64 [[RESULT:v\[[0-9]+:[0-9]+\]]], [[VPTR]], v[[[LOVDATA]]:[[HIVDATA]]] offset:32
; GCN: buffer_store_dwordx2 [[RESULT]],
; GCN: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/long-branch-reserve-register.ll b/llvm/test/CodeGen/AMDGPU/long-branch-reserve-register.ll
index 4d751f2605c39..e856d500f61ab 100644
--- a/llvm/test/CodeGen/AMDGPU/long-branch-reserve-register.ll
+++ b/llvm/test/CodeGen/AMDGPU/long-branch-reserve-register.ll
@@ -14,9 +14,9 @@ declare i32 @llvm.amdgcn.workitem.id.x() #1
define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addrspace(1) %arg, i32 %cnd) #0 {
; GCN-LABEL: uniform_conditional_max_short_forward_branch:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dword s0, s[4:5], 0xb
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s0, 0
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
; GCN-NEXT: s_cbranch_scc1 .LBB0_2
; GCN-NEXT: ; %bb.1: ; %bb2
; GCN-NEXT: ;;#ASMSTART
@@ -26,11 +26,11 @@ define amdgpu_kernel void @uniform_conditional_max_short_forward_branch(ptr addr
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: s_sleep 0
; GCN-NEXT: .LBB0_2: ; %bb3
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
@@ -55,16 +55,16 @@ bb3:
define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrspace(1) %arg, i32 %cnd) #0 {
; GCN-LABEL: uniform_conditional_min_long_forward_branch:
; GCN: ; %bb.0: ; %bb0
-; GCN-NEXT: s_load_dword s0, s[4:5], 0xb
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s0, 0
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
; GCN-NEXT: s_cbranch_scc0 .LBB1_1
; GCN-NEXT: ; %bb.3: ; %bb0
-; GCN-NEXT: s_getpc_b64 s[2:3]
+; GCN-NEXT: s_getpc_b64 s[8:9]
; GCN-NEXT: .Lpost_getpc0:
-; GCN-NEXT: s_add_u32 s2, s2, (.LBB1_2-.Lpost_getpc0)&4294967295
-; GCN-NEXT: s_addc_u32 s3, s3, (.LBB1_2-.Lpost_getpc0)>>32
-; GCN-NEXT: s_setpc_b64 s[2:3]
+; GCN-NEXT: s_add_u32 s8, s8, (.LBB1_2-.Lpost_getpc0)&4294967295
+; GCN-NEXT: s_addc_u32 s9, s9, (.LBB1_2-.Lpost_getpc0)>>32
+; GCN-NEXT: s_setpc_b64 s[8:9]
; GCN-NEXT: .LBB1_1: ; %bb2
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: v_nop_e64
@@ -73,11 +73,11 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_branch(ptr addrs
; GCN-NEXT: v_nop_e64
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: .LBB1_2: ; %bb3
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
@@ -102,10 +102,10 @@ bb3:
define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr addrspace(1) %arg, float %cnd) #0 {
; GCN-LABEL: uniform_conditional_min_long_forward_vcnd_branch:
; GCN: ; %bb.0: ; %bb0
-; GCN-NEXT: s_load_dword s0, s[4:5], 0xb
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_cmp_eq_f32_e64 s[2:3], s0, 0
-; GCN-NEXT: s_and_b64 vcc, exec, s[2:3]
+; GCN-NEXT: v_cmp_eq_f32_e64 s[4:5], s2, 0
+; GCN-NEXT: s_and_b64 vcc, exec, s[4:5]
; GCN-NEXT: s_cbranch_vccz .LBB2_1
; GCN-NEXT: ; %bb.3: ; %bb0
; GCN-NEXT: s_getpc_b64 s[8:9]
@@ -122,11 +122,11 @@ define amdgpu_kernel void @uniform_conditional_min_long_forward_vcnd_branch(ptr
; GCN-NEXT: v_nop_e64
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: .LBB2_2: ; %bb3
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
@@ -254,10 +254,10 @@ bb3:
define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr addrspace(1) %arg, i32 %arg1) {
; GCN-LABEL: uniform_unconditional_min_long_forward_branch:
; GCN: ; %bb.0: ; %bb0
-; GCN-NEXT: s_load_dword s0, s[4:5], 0xb
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s0, 0
-; GCN-NEXT: s_mov_b64 s[0:1], -1
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-NEXT: s_mov_b64 s[2:3], -1
; GCN-NEXT: s_cbranch_scc0 .LBB5_1
; GCN-NEXT: ; %bb.7: ; %bb0
; GCN-NEXT: s_getpc_b64 s[6:7]
@@ -266,7 +266,7 @@ define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr add
; GCN-NEXT: s_addc_u32 s7, s7, (.LBB5_4-.Lpost_getpc5)>>32
; GCN-NEXT: s_setpc_b64 s[6:7]
; GCN-NEXT: .LBB5_1: ; %Flow
-; GCN-NEXT: s_andn2_b64 vcc, exec, s[0:1]
+; GCN-NEXT: s_andn2_b64 vcc, exec, s[2:3]
; GCN-NEXT: s_cbranch_vccnz .LBB5_3
; GCN-NEXT: .LBB5_2: ; %bb2
; GCN-NEXT: s_mov_b32 s3, 0xf000
@@ -275,12 +275,10 @@ define amdgpu_kernel void @uniform_unconditional_min_long_forward_branch(ptr add
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: .LBB5_3: ; %bb4
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_mov_b32 s2, -1
; GCN-NEXT: s_waitcnt expcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, 63
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa.ll b/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa.ll
index 24c1bfb8d50f0..89e4dafdbd8c3 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa.ll
@@ -26,18 +26,17 @@ define amdgpu_kernel void @no_clobber_ds_load_stores_x2(ptr addrspace(1) %arg, i
;
; GCN-LABEL: no_clobber_ds_load_stores_x2:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 1
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: v_mov_b32_e32 v2, 2
; GCN-NEXT: ds_write_b32 v1, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 s0, s0, 2
-; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_lshl_b32 s2, s2, 2
+; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: ds_write_b32 v1, v2 offset:256
; GCN-NEXT: ds_read_b32 v2, v0
; GCN-NEXT: ds_read_b32 v0, v0 offset:256
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_add_u32_e32 v0, v2, v0
; GCN-NEXT: global_store_dword v1, v0, s[0:1]
@@ -74,23 +73,23 @@ define amdgpu_kernel void @no_clobber_ds_load_stores_x3(ptr addrspace(1) %arg, i
;
; GCN-LABEL: no_clobber_ds_load_stores_x3:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: v_mov_b32_e32 v2, 2
; GCN-NEXT: v_mov_b32_e32 v0, 1
; GCN-NEXT: ds_write_b32 v1, v2 offset:256
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 s0, s0, 2
+; GCN-NEXT: s_lshl_b32 s2, s2, 2
; GCN-NEXT: v_mov_b32_e32 v2, 3
; GCN-NEXT: ds_write_b32 v1, v0
-; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: ds_write_b32 v1, v2 offset:512
; GCN-NEXT: ds_read_b32 v2, v0
; GCN-NEXT: ds_read_b32 v3, v0 offset:256
; GCN-NEXT: ds_read_b32 v0, v0 offset:512
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_waitcnt lgkmcnt(1)
; GCN-NEXT: v_add_u32_e32 v2, v2, v3
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_add_u32_e32 v0, v2, v0
; GCN-NEXT: global_store_dword v1, v0, s[0:1]
; GCN-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/lower-lds-with-alias-scope.ll b/llvm/test/CodeGen/AMDGPU/lower-lds-with-alias-scope.ll
index d8d7fc1d7a3bd..36af28790db09 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-lds-with-alias-scope.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-lds-with-alias-scope.ll
@@ -8,16 +8,15 @@
define amdgpu_kernel void @ds_load_stores_aainfo(ptr addrspace(1) %arg, i32 %i) {
; GCN-LABEL: ds_load_stores_aainfo:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 1
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 s0, s0, 2
-; GCN-NEXT: v_mov_b32_e32 v4, s0
+; GCN-NEXT: s_lshl_b32 s2, s2, 2
+; GCN-NEXT: v_mov_b32_e32 v4, s2
; GCN-NEXT: ds_read2_b32 v[2:3], v4 offset1:1
; GCN-NEXT: ds_write_b64 v1, v[0:1] offset:512
; GCN-NEXT: ds_read2_b32 v[4:5], v4 offset0:64 offset1:65
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(1) SyncID(0)
diff --git a/llvm/test/CodeGen/AMDGPU/lower-lds-with-noalias.ll b/llvm/test/CodeGen/AMDGPU/lower-lds-with-noalias.ll
index 0d0daeaae547d..584adfc4806b7 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-lds-with-noalias.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-lds-with-noalias.ll
@@ -10,16 +10,15 @@
define amdgpu_kernel void @ds_load_stores_aainfo(ptr addrspace(1) %arg, i32 %i) {
; GCN-LABEL: ds_load_stores_aainfo:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 1
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 s0, s0, 2
-; GCN-NEXT: v_mov_b32_e32 v4, s0
+; GCN-NEXT: s_lshl_b32 s2, s2, 2
+; GCN-NEXT: v_mov_b32_e32 v4, s2
; GCN-NEXT: ds_read2_b32 v[2:3], v4 offset1:1
; GCN-NEXT: ds_write_b64 v1, v[0:1] offset:512
; GCN-NEXT: ds_read2_b32 v[4:5], v4 offset0:64 offset1:65
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000200) size(1) SyncID(0)
; GCN-NEXT: ; sched_group_barrier mask(0x00000100) size(1) SyncID(0)
diff --git a/llvm/test/CodeGen/AMDGPU/lshl64-to-32.ll b/llvm/test/CodeGen/AMDGPU/lshl64-to-32.ll
index 4187a38ccc8ec..ccd0c686451b2 100644
--- a/llvm/test/CodeGen/AMDGPU/lshl64-to-32.ll
+++ b/llvm/test/CodeGen/AMDGPU/lshl64-to-32.ll
@@ -4,15 +4,16 @@
define amdgpu_kernel void @zext_shl64_to_32(ptr addrspace(1) nocapture %out, i32 %x) {
; GCN-LABEL: zext_shl64_to_32:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_lshl_b32 s4, s6, 2
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_lshl_b32 s0, s2, 2
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-NEXT: s_endpgm
%and = and i32 %x, 1073741823
%ext = zext i32 %and to i64
@@ -24,16 +25,17 @@ define amdgpu_kernel void @zext_shl64_to_32(ptr addrspace(1) nocapture %out, i32
define amdgpu_kernel void @sext_shl64_to_32(ptr addrspace(1) nocapture %out, i32 %x) {
; GCN-LABEL: sext_shl64_to_32:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_and_b32 s4, s6, 0x1fffffff
-; GCN-NEXT: s_lshl_b32 s4, s4, 2
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_and_b32 s0, s2, 0x1fffffff
+; GCN-NEXT: s_lshl_b32 s0, s0, 2
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-NEXT: s_endpgm
%and = and i32 %x, 536870911
%ext = sext i32 %and to i64
@@ -45,17 +47,18 @@ define amdgpu_kernel void @sext_shl64_to_32(ptr addrspace(1) nocapture %out, i32
define amdgpu_kernel void @zext_shl64_overflow(ptr addrspace(1) nocapture %out, i32 %x) {
; GCN-LABEL: zext_shl64_overflow:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s5, 0
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_and_b32 s4, s6, 0x7fffffff
-; GCN-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_and_b32 s0, s2, 0x7fffffff
+; GCN-NEXT: s_mov_b32 s1, 0
+; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-NEXT: s_endpgm
%and = and i32 %x, 2147483647
%ext = zext i32 %and to i64
@@ -67,17 +70,18 @@ define amdgpu_kernel void @zext_shl64_overflow(ptr addrspace(1) nocapture %out,
define amdgpu_kernel void @sext_shl64_overflow(ptr addrspace(1) nocapture %out, i32 %x) {
; GCN-LABEL: sext_shl64_overflow:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s5, 0
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_and_b32 s4, s6, 0x7fffffff
-; GCN-NEXT: s_lshl_b64 s[4:5], s[4:5], 2
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_and_b32 s0, s2, 0x7fffffff
+; GCN-NEXT: s_mov_b32 s1, 0
+; GCN-NEXT: s_lshl_b64 s[0:1], s[0:1], 2
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-NEXT: s_endpgm
%and = and i32 %x, 2147483647
%ext = sext i32 %and to i64
diff --git a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
index 9056d40ad8878..128325c3a0bfe 100644
--- a/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/lshr.v2i16.ll
@@ -20,35 +20,36 @@ define amdgpu_kernel void @s_lshr_v2i16(ptr addrspace(1) %out, <2 x i16> %lhs, <
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s4, s3, 16
-; VI-NEXT: s_lshr_b32 s5, s2, 16
-; VI-NEXT: s_and_b32 s2, s2, 0xffff
-; VI-NEXT: s_lshr_b32 s4, s5, s4
-; VI-NEXT: s_lshr_b32 s2, s2, s3
-; VI-NEXT: s_lshl_b32 s3, s4, 16
-; VI-NEXT: s_or_b32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_lshr_b32 s0, s3, 16
+; VI-NEXT: s_lshr_b32 s1, s2, 16
+; VI-NEXT: s_and_b32 s2, s2, 0xffff
+; VI-NEXT: s_lshr_b32 s0, s1, s0
+; VI-NEXT: s_lshl_b32 s0, s0, 16
+; VI-NEXT: s_lshr_b32 s1, s2, s3
+; VI-NEXT: s_or_b32 s0, s1, s0
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; CI-LABEL: s_lshr_v2i16:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; CI-NEXT: s_and_b32 s6, s4, 0xffff
-; CI-NEXT: s_lshr_b32 s4, s4, 16
-; CI-NEXT: s_lshr_b32 s7, s5, 16
-; CI-NEXT: s_lshr_b32 s4, s4, s7
-; CI-NEXT: s_lshl_b32 s4, s4, 16
-; CI-NEXT: s_lshr_b32 s5, s6, s5
-; CI-NEXT: s_or_b32 s4, s5, s4
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, -1
-; CI-NEXT: v_mov_b32_e32 v0, s4
-; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; CI-NEXT: s_mov_b32 s4, s0
+; CI-NEXT: s_mov_b32 s5, s1
+; CI-NEXT: s_and_b32 s0, s2, 0xffff
+; CI-NEXT: s_lshr_b32 s1, s2, 16
+; CI-NEXT: s_lshr_b32 s2, s3, 16
+; CI-NEXT: s_lshr_b32 s1, s1, s2
+; CI-NEXT: s_lshl_b32 s1, s1, 16
+; CI-NEXT: s_lshr_b32 s0, s0, s3
+; CI-NEXT: s_or_b32 s0, s0, s1
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
;
; GFX10-LABEL: s_lshr_v2i16:
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-fence.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-fence.ll
index 7885a9317748b..bc0df3e0c6999 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-fence.ll
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-fence.ll
@@ -7,21 +7,19 @@ define amdgpu_kernel void @test_fence(ptr addrspace(1) %arg, i1 %arg1) {
; CHECK-LABEL: test_fence:
; CHECK: ; %bb.0: ; %bb
; CHECK-NEXT: v_mov_b32_e32 v1, 0
-; CHECK-NEXT: s_load_b32 s0, s[4:5], 0x2c
+; CHECK-NEXT: s_load_b96 s[0:2], s[4:5], 0x24
; CHECK-NEXT: ds_load_b32 v1, v1
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: s_bitcmp0_b32 s0, 0
+; CHECK-NEXT: s_bitcmp0_b32 s2, 0
; CHECK-NEXT: s_wait_dscnt 0x0
; CHECK-NEXT: s_barrier_signal -1
; CHECK-NEXT: s_barrier_wait -1
; CHECK-NEXT: global_inv scope:SCOPE_SE
; CHECK-NEXT: s_cbranch_scc1 .LBB0_2
; CHECK-NEXT: ; %bb.1: ; %bb2
-; CHECK-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
; CHECK-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: global_atomic_add_u32 v0, v1, s[0:1] scope:SCOPE_SYS
; CHECK-NEXT: .LBB0_2: ; %bb8
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
index 4b658de512f81..c1ca69b02b44a 100644
--- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
+++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll
@@ -469,28 +469,27 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no
; CHECK-NEXT: s_xor_b32 s4, exec_lo, s4
; CHECK-NEXT: s_cbranch_execz .LBB0_31
; CHECK-NEXT: ; %bb.30: ; in Loop: Header=BB0_28 Depth=1
-; CHECK-NEXT: v_xor_b32_e32 v5, v60, v58
-; CHECK-NEXT: v_lshrrev_b64 v[3:4], 16, v[45:46]
+; CHECK-NEXT: v_xor_b32_e32 v2, v61, v59
+; CHECK-NEXT: v_xor_b32_e32 v1, v60, v58
; CHECK-NEXT: v_mad_u64_u32 v[7:8], null, 0x180, v73, s[66:67]
-; CHECK-NEXT: v_lshlrev_b32_e32 v0, 5, v0
-; CHECK-NEXT: v_lshrrev_b64 v[1:2], 16, v[56:57]
-; CHECK-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; CHECK-NEXT: v_lshlrev_b32_e32 v12, 5, v0
+; CHECK-NEXT: v_lshrrev_b64 v[5:6], 16, v[56:57]
+; CHECK-NEXT: v_lshlrev_b32_e32 v4, 16, v45
; CHECK-NEXT: v_lshlrev_b32_e32 v9, 6, v72
-; CHECK-NEXT: v_lshlrev_b32_e32 v11, 12, v63
-; CHECK-NEXT: v_xor_b32_e32 v6, v61, v59
+; CHECK-NEXT: v_lshrrev_b64 v[2:3], 16, v[1:2]
+; CHECK-NEXT: v_lshlrev_b32_e32 v10, 12, v63
+; CHECK-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; CHECK-NEXT: v_lshrrev_b64 v[0:1], 16, v[45:46]
+; CHECK-NEXT: v_add_co_u32 v7, vcc_lo, v7, v12
+; CHECK-NEXT: v_or_b32_e32 v6, v4, v6
+; CHECK-NEXT: v_or3_b32 v4, v9, v10, v62
+; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, 0, v8, vcc_lo
+; CHECK-NEXT: v_or_b32_e32 v1, v11, v1
+; CHECK-NEXT: global_store_dwordx3 v[7:8], v[4:6], off offset:4
+; CHECK-NEXT: global_store_dwordx4 v[7:8], v[0:3], off offset:16
; CHECK-NEXT: ; implicit-def: $vgpr42
; CHECK-NEXT: ; implicit-def: $vgpr43
; CHECK-NEXT: ; implicit-def: $vgpr44
-; CHECK-NEXT: v_add_co_u32 v7, vcc_lo, v7, v0
-; CHECK-NEXT: v_or_b32_e32 v4, v10, v4
-; CHECK-NEXT: v_lshlrev_b32_e32 v10, 16, v45
-; CHECK-NEXT: v_or3_b32 v9, v9, v11, v62
-; CHECK-NEXT: v_add_co_ci_u32_e64 v8, null, 0, v8, vcc_lo
-; CHECK-NEXT: v_lshrrev_b64 v[5:6], 16, v[5:6]
-; CHECK-NEXT: v_or_b32_e32 v2, v10, v2
-; CHECK-NEXT: global_store_dword v[7:8], v9, off offset:4
-; CHECK-NEXT: global_store_dwordx4 v[7:8], v[1:4], off offset:8
-; CHECK-NEXT: global_store_dwordx2 v[7:8], v[5:6], off offset:24
; CHECK-NEXT: .LBB0_31: ; %Flow
; CHECK-NEXT: ; in Loop: Header=BB0_28 Depth=1
; CHECK-NEXT: s_andn2_saveexec_b32 s4, s4
diff --git a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
index 1c4484934f27e..79c432f852ee2 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_64_32.ll
@@ -1036,14 +1036,14 @@ define i64 @mad_i64_i32_unpack_i64ops(i64 %arg0) #0 {
define amdgpu_kernel void @mad_i64_i32_uniform(ptr addrspace(1) %out, i32 %arg0, i32 %arg1, i64 %arg2) #0 {
; CI-LABEL: mad_i64_i32_uniform:
; CI: ; %bb.0:
+; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_mov_b32_e32 v0, s8
+; CI-NEXT: v_mov_b32_e32 v1, s9
; CI-NEXT: v_mov_b32_e32 v2, s3
-; CI-NEXT: v_mov_b32_e32 v0, s4
-; CI-NEXT: v_mov_b32_e32 v1, s5
; CI-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v2, v[0:1]
; CI-NEXT: s_mov_b32 s4, s0
; CI-NEXT: s_mov_b32 s5, s1
diff --git a/llvm/test/CodeGen/AMDGPU/mad_int24.ll b/llvm/test/CodeGen/AMDGPU/mad_int24.ll
index dd883104cd799..014bd5886b251 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_int24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_int24.ll
@@ -8,30 +8,34 @@
define amdgpu_kernel void @i32_mad24(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
; GCN-LABEL: i32_mad24:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_load_dword s4, s[4:5], 0xd
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_bfe_i32 s0, s0, 0x180000
-; GCN-NEXT: s_bfe_i32 s1, s1, 0x180000
-; GCN-NEXT: s_mul_i32 s0, s0, s1
-; GCN-NEXT: s_add_i32 s0, s0, s2
+; GCN-NEXT: s_bfe_i32 s2, s2, 0x180000
+; GCN-NEXT: s_bfe_i32 s3, s3, 0x180000
+; GCN-NEXT: s_mul_i32 s2, s2, s3
+; GCN-NEXT: s_add_i32 s2, s2, s4
; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
;
; VI-LABEL: i32_mad24:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dword s8, s[4:5], 0x34
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfe_i32 s0, s0, 0x180000
-; VI-NEXT: s_bfe_i32 s1, s1, 0x180000
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_bfe_i32 s0, s2, 0x180000
+; VI-NEXT: s_bfe_i32 s1, s3, 0x180000
; VI-NEXT: s_mul_i32 s0, s0, s1
-; VI-NEXT: s_add_i32 s0, s0, s2
+; VI-NEXT: s_add_i32 s0, s0, s8
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/mad_uint24.ll b/llvm/test/CodeGen/AMDGPU/mad_uint24.ll
index 05bd3ac93d608..1484de8d77338 100644
--- a/llvm/test/CodeGen/AMDGPU/mad_uint24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad_uint24.ll
@@ -43,30 +43,34 @@ define amdgpu_kernel void @u32_mad24(ptr addrspace(1) %out, i32 %a, i32 %b, i32
;
; GCN-LABEL: u32_mad24:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_load_dword s4, s[4:5], 0xd
; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_and_b32 s0, s0, 0xffffff
-; GCN-NEXT: s_and_b32 s1, s1, 0xffffff
-; GCN-NEXT: s_mul_i32 s0, s0, s1
-; GCN-NEXT: s_add_i32 s0, s0, s2
+; GCN-NEXT: s_and_b32 s2, s2, 0xffffff
+; GCN-NEXT: s_and_b32 s3, s3, 0xffffff
+; GCN-NEXT: s_mul_i32 s2, s2, s3
+; GCN-NEXT: s_add_i32 s2, s2, s4
; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
;
; GFX8-LABEL: u32_mad24:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dword s8, s[4:5], 0x34
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s0, s0, 0xffffff
-; GFX8-NEXT: s_and_b32 s1, s1, 0xffffff
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: s_and_b32 s0, s2, 0xffffff
+; GFX8-NEXT: s_and_b32 s1, s3, 0xffffff
; GFX8-NEXT: s_mul_i32 s0, s0, s1
-; GFX8-NEXT: s_add_i32 s0, s0, s2
+; GFX8-NEXT: s_add_i32 s0, s0, s8
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
@@ -133,31 +137,33 @@ define amdgpu_kernel void @i16_mad24(ptr addrspace(1) %out, i16 %a, i16 %b, i16
; GCN-LABEL: i16_mad24:
; GCN: ; %bb.0: ; %entry
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_lshr_b32 s2, s4, 16
-; GCN-NEXT: s_mul_i32 s2, s4, s2
-; GCN-NEXT: s_add_i32 s2, s2, s5
-; GCN-NEXT: s_sext_i32_i16 s4, s2
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_lshr_b32 s4, s2, 16
+; GCN-NEXT: s_mul_i32 s2, s2, s4
+; GCN-NEXT: s_add_i32 s2, s2, s3
+; GCN-NEXT: s_sext_i32_i16 s2, s2
+; GCN-NEXT: s_mov_b32 s6, -1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s2
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
;
; GFX8-LABEL: i16_mad24:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX8-NEXT: s_lshr_b32 s6, s4, 16
-; GFX8-NEXT: s_mul_i32 s4, s4, s6
-; GFX8-NEXT: s_add_i32 s4, s4, s5
-; GFX8-NEXT: s_sext_i32_i16 s4, s4
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_lshr_b32 s0, s2, 16
+; GFX8-NEXT: s_mul_i32 s0, s2, s0
+; GFX8-NEXT: s_add_i32 s0, s0, s3
+; GFX8-NEXT: s_sext_i32_i16 s0, s0
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
entry:
%0 = mul i16 %a, %b
@@ -218,10 +224,9 @@ define amdgpu_kernel void @i8_mad24(ptr addrspace(1) %out, i8 %a, i8 %b, i8 %c)
;
; GCN-LABEL: i8_mad24:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_lshr_b32 s4, s2, 8
; GCN-NEXT: s_lshr_b32 s5, s2, 16
; GCN-NEXT: s_mul_i32 s2, s2, s4
@@ -234,18 +239,19 @@ define amdgpu_kernel void @i8_mad24(ptr addrspace(1) %out, i8 %a, i8 %b, i8 %c)
;
; GFX8-LABEL: i8_mad24:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_lshr_b32 s4, s6, 8
-; GFX8-NEXT: s_lshr_b32 s5, s6, 16
-; GFX8-NEXT: s_mul_i32 s4, s6, s4
-; GFX8-NEXT: s_add_i32 s4, s4, s5
-; GFX8-NEXT: s_sext_i32_i8 s4, s4
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_lshr_b32 s0, s2, 8
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: s_lshr_b32 s1, s2, 16
+; GFX8-NEXT: s_mul_i32 s0, s2, s0
+; GFX8-NEXT: s_add_i32 s0, s0, s1
+; GFX8-NEXT: s_sext_i32_i8 s0, s0
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
entry:
%0 = mul i8 %a, %b
@@ -300,16 +306,15 @@ define amdgpu_kernel void @i24_i32_i32_mad(ptr addrspace(1) %out, i32 %a, i32 %b
;
; GCN-LABEL: i24_i32_i32_mad:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s2, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_mov_b32 s3, 0xf000
; GCN-NEXT: s_ashr_i32 s2, s2, 8
-; GCN-NEXT: s_cmp_lg_u32 s6, 0
+; GCN-NEXT: s_cmp_lg_u32 s4, 0
; GCN-NEXT: s_cselect_b32 s2, s2, 34
-; GCN-NEXT: s_mul_i32 s2, s2, s6
-; GCN-NEXT: s_add_i32 s4, s2, s7
+; GCN-NEXT: s_mul_i32 s2, s2, s4
+; GCN-NEXT: s_add_i32 s4, s2, s5
; GCN-NEXT: s_mov_b32 s2, -1
; GCN-NEXT: v_mov_b32_e32 v0, s4
; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -317,19 +322,20 @@ define amdgpu_kernel void @i24_i32_i32_mad(ptr addrspace(1) %out, i32 %a, i32 %b
;
; GFX8-LABEL: i24_i32_i32_mad:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dword s8, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_ashr_i32 s4, s8, 8
-; GFX8-NEXT: s_cmp_lg_u32 s6, 0
-; GFX8-NEXT: s_cselect_b32 s4, s4, 34
-; GFX8-NEXT: s_mul_i32 s4, s4, s6
-; GFX8-NEXT: s_add_i32 s4, s4, s7
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_ashr_i32 s0, s2, 8
+; GFX8-NEXT: s_cmp_lg_u32 s8, 0
+; GFX8-NEXT: s_cselect_b32 s0, s0, 34
+; GFX8-NEXT: s_mul_i32 s0, s0, s8
+; GFX8-NEXT: s_add_i32 s0, s0, s9
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
entry:
%0 = ashr i32 %a, 8
@@ -423,56 +429,52 @@ define amdgpu_kernel void @extra_and(ptr addrspace(1) %arg, i32 %arg2, i32 %arg3
;
; GCN-LABEL: extra_and:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb
-; GCN-NEXT: s_mov_b32 s2, 0
-; GCN-NEXT: s_mov_b32 s6, 0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s4, 0
+; GCN-NEXT: s_mov_b32 s5, 0
; GCN-NEXT: .LBB4_1: ; %bb4
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: s_and_b32 s3, s6, 0xffffff
-; GCN-NEXT: s_and_b32 s6, s6, 0xffffff
-; GCN-NEXT: s_and_b32 s2, s2, 0xffffff
-; GCN-NEXT: s_mul_i32 s3, s3, s2
-; GCN-NEXT: s_mul_i32 s6, s6, s2
+; GCN-NEXT: s_and_b32 s6, s5, 0xffffff
+; GCN-NEXT: s_and_b32 s5, s5, 0xffffff
+; GCN-NEXT: s_and_b32 s4, s4, 0xffffff
+; GCN-NEXT: s_mul_i32 s6, s6, s4
+; GCN-NEXT: s_mul_i32 s5, s5, s4
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_add_i32 s2, s0, s3
-; GCN-NEXT: s_add_i32 s6, s1, s6
-; GCN-NEXT: s_add_i32 s3, s2, s6
-; GCN-NEXT: s_cmp_lg_u32 s3, 8
+; GCN-NEXT: s_add_i32 s4, s2, s6
+; GCN-NEXT: s_add_i32 s5, s3, s5
+; GCN-NEXT: s_add_i32 s6, s4, s5
+; GCN-NEXT: s_cmp_lg_u32 s6, 8
; GCN-NEXT: s_cbranch_scc1 .LBB4_1
; GCN-NEXT: ; %bb.2: ; %bb18
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s7, 0xf000
-; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s3
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
; GFX8-LABEL: extra_and:
; GFX8: ; %bb.0: ; %bb
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
-; GFX8-NEXT: s_mov_b32 s2, 0
-; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s4, 0
+; GFX8-NEXT: s_mov_b32 s5, 0
; GFX8-NEXT: .LBB4_1: ; %bb4
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_and_b32 s3, s6, 0xffffff
-; GFX8-NEXT: s_and_b32 s6, s6, 0xffffff
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffffff
-; GFX8-NEXT: s_mul_i32 s3, s3, s2
-; GFX8-NEXT: s_mul_i32 s6, s6, s2
+; GFX8-NEXT: s_and_b32 s6, s5, 0xffffff
+; GFX8-NEXT: s_and_b32 s5, s5, 0xffffff
+; GFX8-NEXT: s_and_b32 s4, s4, 0xffffff
+; GFX8-NEXT: s_mul_i32 s6, s6, s4
+; GFX8-NEXT: s_mul_i32 s5, s5, s4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_add_i32 s2, s0, s3
-; GFX8-NEXT: s_add_i32 s6, s1, s6
-; GFX8-NEXT: s_add_i32 s3, s2, s6
-; GFX8-NEXT: s_cmp_lg_u32 s3, 8
+; GFX8-NEXT: s_add_i32 s4, s2, s6
+; GFX8-NEXT: s_add_i32 s5, s3, s5
+; GFX8-NEXT: s_add_i32 s6, s4, s5
+; GFX8-NEXT: s_cmp_lg_u32 s6, 8
; GFX8-NEXT: s_cbranch_scc1 .LBB4_1
; GFX8-NEXT: ; %bb.2: ; %bb18
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mov_b32 s6, -1
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
bb:
br label %bb4
@@ -581,56 +583,52 @@ define amdgpu_kernel void @dont_remove_shift(ptr addrspace(1) %arg, i32 %arg2, i
;
; GCN-LABEL: dont_remove_shift:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb
-; GCN-NEXT: s_mov_b32 s2, 0
-; GCN-NEXT: s_mov_b32 s6, 0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s4, 0
+; GCN-NEXT: s_mov_b32 s5, 0
; GCN-NEXT: .LBB5_1: ; %bb4
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: s_lshr_b32 s3, s6, 8
-; GCN-NEXT: s_lshr_b32 s6, s6, 8
-; GCN-NEXT: s_lshr_b32 s2, s2, 8
-; GCN-NEXT: s_mul_i32 s3, s3, s2
-; GCN-NEXT: s_mul_i32 s6, s6, s2
+; GCN-NEXT: s_lshr_b32 s6, s5, 8
+; GCN-NEXT: s_lshr_b32 s5, s5, 8
+; GCN-NEXT: s_lshr_b32 s4, s4, 8
+; GCN-NEXT: s_mul_i32 s6, s6, s4
+; GCN-NEXT: s_mul_i32 s5, s5, s4
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_add_i32 s2, s0, s3
-; GCN-NEXT: s_add_i32 s6, s1, s6
-; GCN-NEXT: s_add_i32 s3, s2, s6
-; GCN-NEXT: s_cmp_lg_u32 s3, 8
+; GCN-NEXT: s_add_i32 s4, s2, s6
+; GCN-NEXT: s_add_i32 s5, s3, s5
+; GCN-NEXT: s_add_i32 s6, s4, s5
+; GCN-NEXT: s_cmp_lg_u32 s6, 8
; GCN-NEXT: s_cbranch_scc1 .LBB5_1
; GCN-NEXT: ; %bb.2: ; %bb18
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s7, 0xf000
-; GCN-NEXT: s_mov_b32 s6, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s3
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GCN-NEXT: s_mov_b32 s3, 0xf000
+; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: v_mov_b32_e32 v0, s6
+; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GCN-NEXT: s_endpgm
;
; GFX8-LABEL: dont_remove_shift:
; GFX8: ; %bb.0: ; %bb
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
-; GFX8-NEXT: s_mov_b32 s2, 0
-; GFX8-NEXT: s_mov_b32 s6, 0
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s4, 0
+; GFX8-NEXT: s_mov_b32 s5, 0
; GFX8-NEXT: .LBB5_1: ; %bb4
; GFX8-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX8-NEXT: s_lshr_b32 s3, s6, 8
-; GFX8-NEXT: s_lshr_b32 s6, s6, 8
-; GFX8-NEXT: s_lshr_b32 s2, s2, 8
-; GFX8-NEXT: s_mul_i32 s3, s3, s2
-; GFX8-NEXT: s_mul_i32 s6, s6, s2
+; GFX8-NEXT: s_lshr_b32 s6, s5, 8
+; GFX8-NEXT: s_lshr_b32 s5, s5, 8
+; GFX8-NEXT: s_lshr_b32 s4, s4, 8
+; GFX8-NEXT: s_mul_i32 s6, s6, s4
+; GFX8-NEXT: s_mul_i32 s5, s5, s4
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_add_i32 s2, s0, s3
-; GFX8-NEXT: s_add_i32 s6, s1, s6
-; GFX8-NEXT: s_add_i32 s3, s2, s6
-; GFX8-NEXT: s_cmp_lg_u32 s3, 8
+; GFX8-NEXT: s_add_i32 s4, s2, s6
+; GFX8-NEXT: s_add_i32 s5, s3, s5
+; GFX8-NEXT: s_add_i32 s6, s4, s5
+; GFX8-NEXT: s_cmp_lg_u32 s6, 8
; GFX8-NEXT: s_cbranch_scc1 .LBB5_1
; GFX8-NEXT: ; %bb.2: ; %bb18
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s7, 0xf000
-; GFX8-NEXT: s_mov_b32 s6, -1
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; GFX8-NEXT: s_mov_b32 s3, 0xf000
+; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX8-NEXT: s_endpgm
bb:
br label %bb4
diff --git a/llvm/test/CodeGen/AMDGPU/madak.ll b/llvm/test/CodeGen/AMDGPU/madak.ll
index b16dad4316db4..cc97d4ccf3ce6 100644
--- a/llvm/test/CodeGen/AMDGPU/madak.ll
+++ b/llvm/test/CodeGen/AMDGPU/madak.ll
@@ -772,10 +772,11 @@ define amdgpu_kernel void @v_s_madak_f32(ptr addrspace(1) noalias %out, float %a
; GFX6-NEXT: v_mov_b32_e32 v1, 0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; GFX6-NEXT: v_mov_b32_e32 v3, 0x41200000
-; GFX6-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_mov_b64 s[0:1], s[4:5]
+; GFX6-NEXT: s_waitcnt vmcnt(0)
; GFX6-NEXT: v_mac_f32_e32 v3, s6, v2
; GFX6-NEXT: buffer_store_dword v3, v[0:1], s[0:3], 0 addr64
; GFX6-NEXT: s_endpgm
@@ -789,8 +790,7 @@ define amdgpu_kernel void @v_s_madak_f32(ptr addrspace(1) noalias %out, float %a
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; GFX8-NEXT: flat_load_dword v3, v[0:1]
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, s0, v2
@@ -808,8 +808,7 @@ define amdgpu_kernel void @v_s_madak_f32(ptr addrspace(1) noalias %out, float %a
; GFX9-NEXT: v_mov_b32_e32 v2, 0x41200000
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mac_f32_e32 v2, s2, v1
; GFX9-NEXT: global_store_dword v0, v2, s[0:1]
@@ -821,10 +820,8 @@ define amdgpu_kernel void @v_s_madak_f32(ptr addrspace(1) noalias %out, float %a
; GFX10-MAD-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-MAD-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-MAD-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX10-MAD-NEXT: s_clause 0x1
-; GFX10-MAD-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-MAD-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-MAD-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-MAD-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-MAD-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-MAD-NEXT: v_madak_f32 v1, s2, v1, 0x41200000
; GFX10-MAD-NEXT: global_store_dword v0, v1, s[0:1]
@@ -838,9 +835,7 @@ define amdgpu_kernel void @v_s_madak_f32(ptr addrspace(1) noalias %out, float %a
; GFX11-MAD-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-MAD-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-MAD-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX11-MAD-NEXT: s_clause 0x1
-; GFX11-MAD-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-MAD-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-MAD-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-MAD-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-MAD-NEXT: v_mul_f32_e32 v1, s2, v1
; GFX11-MAD-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -856,8 +851,7 @@ define amdgpu_kernel void @v_s_madak_f32(ptr addrspace(1) noalias %out, float %a
; GFX942-FMA-NEXT: v_mov_b32_e32 v2, 0x41200000
; GFX942-FMA-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-FMA-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX942-FMA-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX942-FMA-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX942-FMA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-FMA-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-FMA-NEXT: v_fmac_f32_e32 v2, s2, v1
; GFX942-FMA-NEXT: global_store_dword v0, v2, s[0:1]
@@ -869,10 +863,8 @@ define amdgpu_kernel void @v_s_madak_f32(ptr addrspace(1) noalias %out, float %a
; GFX10-FMA-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-FMA-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-FMA-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX10-FMA-NEXT: s_clause 0x1
-; GFX10-FMA-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-FMA-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-FMA-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-FMA-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-FMA-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-FMA-NEXT: v_fmaak_f32 v1, s2, v1, 0x41200000
; GFX10-FMA-NEXT: global_store_dword v0, v1, s[0:1]
@@ -886,9 +878,7 @@ define amdgpu_kernel void @v_s_madak_f32(ptr addrspace(1) noalias %out, float %a
; GFX11-FMA-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-FMA-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FMA-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX11-FMA-NEXT: s_clause 0x1
-; GFX11-FMA-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FMA-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FMA-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FMA-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-FMA-NEXT: v_fmaak_f32 v1, s2, v1, 0x41200000
; GFX11-FMA-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -925,10 +915,10 @@ define amdgpu_kernel void @s_s_madak_f32(ptr addrspace(1) %out, float %a, float
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: v_mov_b32_e32 v2, 0x41200000
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_mov_b32_e32 v0, s3
-; GFX8-NEXT: v_mac_f32_e32 v2, s2, v0
+; GFX8-NEXT: v_mov_b32_e32 v3, s3
; GFX8-NEXT: v_mov_b32_e32 v0, s0
; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_mac_f32_e32 v2, s2, v3
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/max.ll b/llvm/test/CodeGen/AMDGPU/max.ll
index 84ef83d47a8db..b2e31e7cef294 100644
--- a/llvm/test/CodeGen/AMDGPU/max.ll
+++ b/llvm/test/CodeGen/AMDGPU/max.ll
@@ -157,13 +157,14 @@ define amdgpu_kernel void @s_test_imax_sge_i32(ptr addrspace(1) %out, i32 %a, i3
; SI-LABEL: s_test_imax_sge_i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_max_i32 s4, s4, s5
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_max_i32 s0, s2, s3
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; GFX1250-LABEL: s_test_imax_sge_i32:
@@ -197,14 +198,15 @@ define amdgpu_kernel void @s_test_imax_sge_i32(ptr addrspace(1) %out, i32 %a, i3
define amdgpu_kernel void @s_test_imax_sge_imm_i32(ptr addrspace(1) %out, i32 %a) nounwind {
; SI-LABEL: s_test_imax_sge_imm_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_max_i32 s4, s6, 9
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_max_i32 s0, s2, 9
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; GFX1250-LABEL: s_test_imax_sge_imm_i32:
@@ -320,14 +322,15 @@ define amdgpu_kernel void @v_test_imax_sge_i8(ptr addrspace(1) %out, ptr addrspa
define amdgpu_kernel void @s_test_imax_sgt_imm_i32(ptr addrspace(1) %out, i32 %a) nounwind {
; SI-LABEL: s_test_imax_sgt_imm_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_max_i32 s4, s6, 9
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_max_i32 s0, s2, 9
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; GFX1250-LABEL: s_test_imax_sgt_imm_i32:
@@ -362,15 +365,16 @@ define amdgpu_kernel void @s_test_imax_sgt_imm_v2i32(ptr addrspace(1) %out, <2 x
; SI-LABEL: s_test_imax_sgt_imm_v2i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_max_i32 s5, s5, 9
-; SI-NEXT: s_max_i32 s4, s4, 9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_max_i32 s0, s3, 9
+; SI-NEXT: s_max_i32 s1, s2, 9
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: v_mov_b32_e32 v1, s0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; GFX1250-LABEL: s_test_imax_sgt_imm_v2i32:
@@ -478,13 +482,14 @@ define amdgpu_kernel void @s_test_imax_sgt_i32(ptr addrspace(1) %out, i32 %a, i3
; SI-LABEL: s_test_imax_sgt_i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_max_i32 s4, s4, s5
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_max_i32 s0, s2, s3
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; GFX1250-LABEL: s_test_imax_sgt_i32:
@@ -589,13 +594,14 @@ define amdgpu_kernel void @s_test_umax_uge_i32(ptr addrspace(1) %out, i32 %a, i3
; SI-LABEL: s_test_umax_uge_i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_max_u32 s4, s4, s5
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_max_u32 s0, s2, s3
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; GFX1250-LABEL: s_test_umax_uge_i32:
@@ -826,13 +832,14 @@ define amdgpu_kernel void @s_test_umax_ugt_i32(ptr addrspace(1) %out, i32 %a, i3
; SI-LABEL: s_test_umax_ugt_i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_max_u32 s4, s4, s5
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_max_u32 s0, s2, s3
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; GFX1250-LABEL: s_test_umax_ugt_i32:
@@ -867,15 +874,16 @@ define amdgpu_kernel void @s_test_umax_ugt_imm_v2i32(ptr addrspace(1) %out, <2 x
; SI-LABEL: s_test_umax_ugt_imm_v2i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_max_u32 s5, s5, 23
-; SI-NEXT: s_max_u32 s4, s4, 15
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_max_u32 s0, s3, 23
+; SI-NEXT: s_max_u32 s1, s2, 15
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: v_mov_b32_e32 v1, s0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; GFX1250-LABEL: s_test_umax_ugt_imm_v2i32:
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
index 5b7c36559a366..5697ea6416b73 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
@@ -162,62 +162,58 @@ define amdgpu_kernel void @memcpy_p0_p5_minsize(ptr %generic, ptr addrspace(5) %
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_mov_b64 s[22:23], s[2:3]
; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1]
-; CHECK-NEXT: s_load_dword s0, s[8:9], 0x8
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s17
; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; CHECK-NEXT: s_add_u32 s20, s20, s17
; CHECK-NEXT: s_addc_u32 s21, s21, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v26, s0
-; CHECK-NEXT: buffer_load_dword v3, v26, s[20:23], 0 offen offset:124
-; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:120
-; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:116
-; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen offset:112
-; CHECK-NEXT: buffer_load_dword v7, v26, s[20:23], 0 offen offset:108
-; CHECK-NEXT: buffer_load_dword v6, v26, s[20:23], 0 offen offset:104
-; CHECK-NEXT: buffer_load_dword v5, v26, s[20:23], 0 offen offset:100
-; CHECK-NEXT: buffer_load_dword v4, v26, s[20:23], 0 offen offset:96
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CHECK-NEXT: buffer_load_dword v11, v26, s[20:23], 0 offen offset:92
-; CHECK-NEXT: buffer_load_dword v10, v26, s[20:23], 0 offen offset:88
-; CHECK-NEXT: buffer_load_dword v9, v26, s[20:23], 0 offen offset:84
-; CHECK-NEXT: buffer_load_dword v8, v26, s[20:23], 0 offen offset:80
-; CHECK-NEXT: buffer_load_dword v15, v26, s[20:23], 0 offen offset:76
-; CHECK-NEXT: buffer_load_dword v14, v26, s[20:23], 0 offen offset:72
-; CHECK-NEXT: buffer_load_dword v13, v26, s[20:23], 0 offen offset:68
-; CHECK-NEXT: buffer_load_dword v12, v26, s[20:23], 0 offen offset:64
-; CHECK-NEXT: buffer_load_dword v16, v26, s[20:23], 0 offen offset:32
-; CHECK-NEXT: buffer_load_dword v17, v26, s[20:23], 0 offen offset:36
-; CHECK-NEXT: buffer_load_dword v18, v26, s[20:23], 0 offen offset:40
-; CHECK-NEXT: buffer_load_dword v19, v26, s[20:23], 0 offen offset:44
-; CHECK-NEXT: buffer_load_dword v20, v26, s[20:23], 0 offen offset:48
-; CHECK-NEXT: buffer_load_dword v21, v26, s[20:23], 0 offen offset:52
-; CHECK-NEXT: buffer_load_dword v22, v26, s[20:23], 0 offen offset:56
-; CHECK-NEXT: buffer_load_dword v23, v26, s[20:23], 0 offen offset:60
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v25, s1
-; CHECK-NEXT: v_mov_b32_e32 v24, s0
-; CHECK-NEXT: s_waitcnt vmcnt(20)
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[0:3] offset:112
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[4:7] offset:96
-; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen
-; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:8
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v4, v26, s[20:23], 0 offen offset:16
-; CHECK-NEXT: buffer_load_dword v5, v26, s[20:23], 0 offen offset:20
-; CHECK-NEXT: buffer_load_dword v6, v26, s[20:23], 0 offen offset:24
-; CHECK-NEXT: buffer_load_dword v7, v26, s[20:23], 0 offen offset:28
-; CHECK-NEXT: buffer_load_dword v3, v26, s[20:23], 0 offen offset:12
+; CHECK-NEXT: v_mov_b32_e32 v25, s2
+; CHECK-NEXT: buffer_load_dword v3, v25, s[20:23], 0 offen offset:92
+; CHECK-NEXT: buffer_load_dword v2, v25, s[20:23], 0 offen offset:88
+; CHECK-NEXT: buffer_load_dword v1, v25, s[20:23], 0 offen offset:84
+; CHECK-NEXT: buffer_load_dword v0, v25, s[20:23], 0 offen offset:80
+; CHECK-NEXT: buffer_load_dword v7, v25, s[20:23], 0 offen offset:76
+; CHECK-NEXT: buffer_load_dword v6, v25, s[20:23], 0 offen offset:72
+; CHECK-NEXT: buffer_load_dword v5, v25, s[20:23], 0 offen offset:68
+; CHECK-NEXT: buffer_load_dword v4, v25, s[20:23], 0 offen offset:64
+; CHECK-NEXT: buffer_load_dword v8, v25, s[20:23], 0 offen
+; CHECK-NEXT: buffer_load_dword v9, v25, s[20:23], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v10, v25, s[20:23], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v11, v25, s[20:23], 0 offen offset:12
+; CHECK-NEXT: buffer_load_dword v12, v25, s[20:23], 0 offen offset:16
+; CHECK-NEXT: buffer_load_dword v13, v25, s[20:23], 0 offen offset:20
+; CHECK-NEXT: buffer_load_dword v14, v25, s[20:23], 0 offen offset:24
+; CHECK-NEXT: buffer_load_dword v15, v25, s[20:23], 0 offen offset:28
+; CHECK-NEXT: buffer_load_dword v16, v25, s[20:23], 0 offen offset:32
+; CHECK-NEXT: buffer_load_dword v17, v25, s[20:23], 0 offen offset:36
+; CHECK-NEXT: buffer_load_dword v18, v25, s[20:23], 0 offen offset:40
+; CHECK-NEXT: buffer_load_dword v19, v25, s[20:23], 0 offen offset:44
+; CHECK-NEXT: buffer_load_dword v20, v25, s[20:23], 0 offen offset:48
+; CHECK-NEXT: buffer_load_dword v21, v25, s[20:23], 0 offen offset:52
+; CHECK-NEXT: buffer_load_dword v22, v25, s[20:23], 0 offen offset:56
+; CHECK-NEXT: buffer_load_dword v23, v25, s[20:23], 0 offen offset:60
+; CHECK-NEXT: buffer_load_dword v27, v25, s[20:23], 0 offen offset:124
+; CHECK-NEXT: buffer_load_dword v26, v25, s[20:23], 0 offen offset:120
+; CHECK-NEXT: buffer_load_dword v24, v25, s[20:23], 0 offen offset:112
+; CHECK-NEXT: buffer_load_dword v31, v25, s[20:23], 0 offen offset:108
+; CHECK-NEXT: buffer_load_dword v30, v25, s[20:23], 0 offen offset:104
+; CHECK-NEXT: buffer_load_dword v29, v25, s[20:23], 0 offen offset:100
+; CHECK-NEXT: buffer_load_dword v28, v25, s[20:23], 0 offen offset:96
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[8:11] offset:80
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[12:15] offset:64
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[20:23] offset:48
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[16:19] offset:32
+; CHECK-NEXT: buffer_load_dword v25, v25, s[20:23], 0 offen offset:116
+; CHECK-NEXT: v_mov_b32_e32 v33, s1
+; CHECK-NEXT: v_mov_b32_e32 v32, s0
+; CHECK-NEXT: s_waitcnt vmcnt(20)
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[8:11]
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[4:7] offset:16
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[0:3]
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[12:15] offset:16
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[16:19] offset:32
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[20:23] offset:48
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[4:7] offset:64
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[0:3] offset:80
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[28:31] offset:96
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[24:27] offset:112
; CHECK-NEXT: s_endpgm
entry:
tail call void @llvm.memcpy.p0.p5.i64(ptr %generic, ptr addrspace(5) %src, i64 128, i1 false)
@@ -450,62 +446,58 @@ define amdgpu_kernel void @memcpy_p0_p5_optsize(ptr %generic, ptr addrspace(5) %
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_mov_b64 s[22:23], s[2:3]
; CHECK-NEXT: s_mov_b64 s[20:21], s[0:1]
-; CHECK-NEXT: s_load_dword s0, s[8:9], 0x8
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s17
; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; CHECK-NEXT: s_add_u32 s20, s20, s17
; CHECK-NEXT: s_addc_u32 s21, s21, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v26, s0
-; CHECK-NEXT: buffer_load_dword v3, v26, s[20:23], 0 offen offset:124
-; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:120
-; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:116
-; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen offset:112
-; CHECK-NEXT: buffer_load_dword v7, v26, s[20:23], 0 offen offset:108
-; CHECK-NEXT: buffer_load_dword v6, v26, s[20:23], 0 offen offset:104
-; CHECK-NEXT: buffer_load_dword v5, v26, s[20:23], 0 offen offset:100
-; CHECK-NEXT: buffer_load_dword v4, v26, s[20:23], 0 offen offset:96
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CHECK-NEXT: buffer_load_dword v11, v26, s[20:23], 0 offen offset:92
-; CHECK-NEXT: buffer_load_dword v10, v26, s[20:23], 0 offen offset:88
-; CHECK-NEXT: buffer_load_dword v9, v26, s[20:23], 0 offen offset:84
-; CHECK-NEXT: buffer_load_dword v8, v26, s[20:23], 0 offen offset:80
-; CHECK-NEXT: buffer_load_dword v15, v26, s[20:23], 0 offen offset:76
-; CHECK-NEXT: buffer_load_dword v14, v26, s[20:23], 0 offen offset:72
-; CHECK-NEXT: buffer_load_dword v13, v26, s[20:23], 0 offen offset:68
-; CHECK-NEXT: buffer_load_dword v12, v26, s[20:23], 0 offen offset:64
-; CHECK-NEXT: buffer_load_dword v16, v26, s[20:23], 0 offen offset:32
-; CHECK-NEXT: buffer_load_dword v17, v26, s[20:23], 0 offen offset:36
-; CHECK-NEXT: buffer_load_dword v18, v26, s[20:23], 0 offen offset:40
-; CHECK-NEXT: buffer_load_dword v19, v26, s[20:23], 0 offen offset:44
-; CHECK-NEXT: buffer_load_dword v20, v26, s[20:23], 0 offen offset:48
-; CHECK-NEXT: buffer_load_dword v21, v26, s[20:23], 0 offen offset:52
-; CHECK-NEXT: buffer_load_dword v22, v26, s[20:23], 0 offen offset:56
-; CHECK-NEXT: buffer_load_dword v23, v26, s[20:23], 0 offen offset:60
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_mov_b32_e32 v25, s1
-; CHECK-NEXT: v_mov_b32_e32 v24, s0
-; CHECK-NEXT: s_waitcnt vmcnt(20)
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[0:3] offset:112
-; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[4:7] offset:96
-; CHECK-NEXT: buffer_load_dword v0, v26, s[20:23], 0 offen
-; CHECK-NEXT: buffer_load_dword v1, v26, s[20:23], 0 offen offset:4
-; CHECK-NEXT: buffer_load_dword v2, v26, s[20:23], 0 offen offset:8
-; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: buffer_load_dword v4, v26, s[20:23], 0 offen offset:16
-; CHECK-NEXT: buffer_load_dword v5, v26, s[20:23], 0 offen offset:20
-; CHECK-NEXT: buffer_load_dword v6, v26, s[20:23], 0 offen offset:24
-; CHECK-NEXT: buffer_load_dword v7, v26, s[20:23], 0 offen offset:28
-; CHECK-NEXT: buffer_load_dword v3, v26, s[20:23], 0 offen offset:12
+; CHECK-NEXT: v_mov_b32_e32 v25, s2
+; CHECK-NEXT: buffer_load_dword v3, v25, s[20:23], 0 offen offset:92
+; CHECK-NEXT: buffer_load_dword v2, v25, s[20:23], 0 offen offset:88
+; CHECK-NEXT: buffer_load_dword v1, v25, s[20:23], 0 offen offset:84
+; CHECK-NEXT: buffer_load_dword v0, v25, s[20:23], 0 offen offset:80
+; CHECK-NEXT: buffer_load_dword v7, v25, s[20:23], 0 offen offset:76
+; CHECK-NEXT: buffer_load_dword v6, v25, s[20:23], 0 offen offset:72
+; CHECK-NEXT: buffer_load_dword v5, v25, s[20:23], 0 offen offset:68
+; CHECK-NEXT: buffer_load_dword v4, v25, s[20:23], 0 offen offset:64
+; CHECK-NEXT: buffer_load_dword v8, v25, s[20:23], 0 offen
+; CHECK-NEXT: buffer_load_dword v9, v25, s[20:23], 0 offen offset:4
+; CHECK-NEXT: buffer_load_dword v10, v25, s[20:23], 0 offen offset:8
+; CHECK-NEXT: buffer_load_dword v11, v25, s[20:23], 0 offen offset:12
+; CHECK-NEXT: buffer_load_dword v12, v25, s[20:23], 0 offen offset:16
+; CHECK-NEXT: buffer_load_dword v13, v25, s[20:23], 0 offen offset:20
+; CHECK-NEXT: buffer_load_dword v14, v25, s[20:23], 0 offen offset:24
+; CHECK-NEXT: buffer_load_dword v15, v25, s[20:23], 0 offen offset:28
+; CHECK-NEXT: buffer_load_dword v16, v25, s[20:23], 0 offen offset:32
+; CHECK-NEXT: buffer_load_dword v17, v25, s[20:23], 0 offen offset:36
+; CHECK-NEXT: buffer_load_dword v18, v25, s[20:23], 0 offen offset:40
+; CHECK-NEXT: buffer_load_dword v19, v25, s[20:23], 0 offen offset:44
+; CHECK-NEXT: buffer_load_dword v20, v25, s[20:23], 0 offen offset:48
+; CHECK-NEXT: buffer_load_dword v21, v25, s[20:23], 0 offen offset:52
+; CHECK-NEXT: buffer_load_dword v22, v25, s[20:23], 0 offen offset:56
+; CHECK-NEXT: buffer_load_dword v23, v25, s[20:23], 0 offen offset:60
+; CHECK-NEXT: buffer_load_dword v27, v25, s[20:23], 0 offen offset:124
+; CHECK-NEXT: buffer_load_dword v26, v25, s[20:23], 0 offen offset:120
+; CHECK-NEXT: buffer_load_dword v24, v25, s[20:23], 0 offen offset:112
+; CHECK-NEXT: buffer_load_dword v31, v25, s[20:23], 0 offen offset:108
+; CHECK-NEXT: buffer_load_dword v30, v25, s[20:23], 0 offen offset:104
+; CHECK-NEXT: buffer_load_dword v29, v25, s[20:23], 0 offen offset:100
+; CHECK-NEXT: buffer_load_dword v28, v25, s[20:23], 0 offen offset:96
; CHECK-NEXT: s_nop 0
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[8:11] offset:80
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[12:15] offset:64
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[20:23] offset:48
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[16:19] offset:32
+; CHECK-NEXT: buffer_load_dword v25, v25, s[20:23], 0 offen offset:116
+; CHECK-NEXT: v_mov_b32_e32 v33, s1
+; CHECK-NEXT: v_mov_b32_e32 v32, s0
+; CHECK-NEXT: s_waitcnt vmcnt(20)
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[8:11]
; CHECK-NEXT: s_waitcnt vmcnt(0)
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[4:7] offset:16
-; CHECK-NEXT: flat_store_dwordx4 v[24:25], v[0:3]
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[12:15] offset:16
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[16:19] offset:32
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[20:23] offset:48
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[4:7] offset:64
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[0:3] offset:80
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[28:31] offset:96
+; CHECK-NEXT: flat_store_dwordx4 v[32:33], v[24:27] offset:112
; CHECK-NEXT: s_endpgm
entry:
tail call void @llvm.memcpy.p0.p5.i64(ptr %generic, ptr addrspace(5) %src, i64 128, i1 false)
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
index fcbbdb6afa146..dcb2bbfe96bd3 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-single-wave-workgroup-memops.ll
@@ -586,33 +586,30 @@ define amdgpu_kernel void @wg_st_seq_cst_single32(ptr addrspace(1) %p, i32 %x) #
; GFX9: bb.0 (%ir-block.0):
; GFX9-NEXT: liveins: $sgpr4_sgpr5
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX9-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: wg_st_seq_cst_single32
; GFX942: bb.0 (%ir-block.0):
; GFX942-NEXT: liveins: $sgpr4_sgpr5
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX942-NEXT: S_ENDPGM 0
;
; GFX10-LABEL: name: wg_st_seq_cst_single32
; GFX10: bb.0 (%ir-block.0):
; GFX10-NEXT: liveins: $sgpr4_sgpr5
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX10-NEXT: S_ENDPGM 0
;
; GFX12-W32-LABEL: name: wg_st_seq_cst_single32
@@ -621,7 +618,7 @@ define amdgpu_kernel void @wg_st_seq_cst_single32(ptr addrspace(1) %p, i32 %x) #
; GFX12-W32-NEXT: {{ $}}
; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX12-W32-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX12-W32-NEXT: S_ENDPGM 0
;
; GFX12-W64-LABEL: name: wg_st_seq_cst_single32
@@ -631,7 +628,7 @@ define amdgpu_kernel void @wg_st_seq_cst_single32(ptr addrspace(1) %p, i32 %x) #
; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX12-W64-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: wg_st_seq_cst_single32
@@ -642,7 +639,7 @@ define amdgpu_kernel void @wg_st_seq_cst_single32(ptr addrspace(1) %p, i32 %x) #
; GFX1250-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM_ec killed renamable $sgpr4_sgpr5, 36, 32 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX1250-NEXT: renamable $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx1250 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
ret void
@@ -653,47 +650,43 @@ define amdgpu_kernel void @wg_st_seq_cst_single64(ptr addrspace(1) %p, i32 %x) #
; GFX9: bb.0 (%ir-block.0):
; GFX9-NEXT: liveins: $sgpr4_sgpr5
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX9-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: wg_st_seq_cst_single64
; GFX942: bb.0 (%ir-block.0):
; GFX942-NEXT: liveins: $sgpr4_sgpr5
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX942-NEXT: S_ENDPGM 0
;
; GFX10-W32-LABEL: name: wg_st_seq_cst_single64
; GFX10-W32: bb.0 (%ir-block.0):
; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
; GFX10-W32-NEXT: S_WAITCNT_lds_direct
; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX10-W32-NEXT: S_ENDPGM 0
;
; GFX10-W64-LABEL: name: wg_st_seq_cst_single64
; GFX10-W64: bb.0 (%ir-block.0):
; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX10-W64-NEXT: S_ENDPGM 0
;
; GFX12-W32-LABEL: name: wg_st_seq_cst_single64
@@ -707,7 +700,7 @@ define amdgpu_kernel void @wg_st_seq_cst_single64(ptr addrspace(1) %p, i32 %x) #
; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX12-W32-NEXT: S_ENDPGM 0
;
; GFX12-W64-LABEL: name: wg_st_seq_cst_single64
@@ -717,7 +710,7 @@ define amdgpu_kernel void @wg_st_seq_cst_single64(ptr addrspace(1) %p, i32 %x) #
; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX12-W64-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: wg_st_seq_cst_single64
@@ -731,7 +724,7 @@ define amdgpu_kernel void @wg_st_seq_cst_single64(ptr addrspace(1) %p, i32 %x) #
; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
ret void
@@ -742,40 +735,37 @@ define amdgpu_kernel void @wg_st_seq_cst_multi(ptr addrspace(1) %p, i32 %x) #2 {
; GFX9: bb.0 (%ir-block.0):
; GFX9-NEXT: liveins: $sgpr4_sgpr5
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX9-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: wg_st_seq_cst_multi
; GFX942: bb.0 (%ir-block.0):
; GFX942-NEXT: liveins: $sgpr4_sgpr5
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 1, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX942-NEXT: S_ENDPGM 0
;
; GFX10-LABEL: name: wg_st_seq_cst_multi
; GFX10: bb.0 (%ir-block.0):
; GFX10-NEXT: liveins: $sgpr4_sgpr5
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX10-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
; GFX10-NEXT: S_WAITCNT_lds_direct
; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX10-NEXT: S_ENDPGM 0
;
; GFX12-W32-LABEL: name: wg_st_seq_cst_multi
@@ -789,7 +779,7 @@ define amdgpu_kernel void @wg_st_seq_cst_multi(ptr addrspace(1) %p, i32 %x) #2 {
; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX12-W32-NEXT: S_ENDPGM 0
;
; GFX12-W64-LABEL: name: wg_st_seq_cst_multi
@@ -804,7 +794,7 @@ define amdgpu_kernel void @wg_st_seq_cst_multi(ptr addrspace(1) %p, i32 %x) #2 {
; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX12-W64-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: wg_st_seq_cst_multi
@@ -818,7 +808,7 @@ define amdgpu_kernel void @wg_st_seq_cst_multi(ptr addrspace(1) %p, i32 %x) #2 {
; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") seq_cst (s32) into %ir.2, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") seq_cst, align 4
ret void
@@ -829,47 +819,43 @@ define amdgpu_kernel void @wg_st_release_single64(ptr addrspace(1) %p, i32 %x) #
; GFX9: bb.0 (%ir-block.0):
; GFX9-NEXT: liveins: $sgpr4_sgpr5
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 1)
; GFX9-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: wg_st_release_single64
; GFX942: bb.0 (%ir-block.0):
; GFX942-NEXT: liveins: $sgpr4_sgpr5
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 1)
; GFX942-NEXT: S_ENDPGM 0
;
; GFX10-W32-LABEL: name: wg_st_release_single64
; GFX10-W32: bb.0 (%ir-block.0):
; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
; GFX10-W32-NEXT: S_WAITCNT_lds_direct
; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 1)
; GFX10-W32-NEXT: S_ENDPGM 0
;
; GFX10-W64-LABEL: name: wg_st_release_single64
; GFX10-W64: bb.0 (%ir-block.0):
; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX10-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 1)
; GFX10-W64-NEXT: S_ENDPGM 0
;
; GFX12-W32-LABEL: name: wg_st_release_single64
@@ -883,7 +869,7 @@ define amdgpu_kernel void @wg_st_release_single64(ptr addrspace(1) %p, i32 %x) #
; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 1)
; GFX12-W32-NEXT: S_ENDPGM 0
;
; GFX12-W64-LABEL: name: wg_st_release_single64
@@ -893,7 +879,7 @@ define amdgpu_kernel void @wg_st_release_single64(ptr addrspace(1) %p, i32 %x) #
; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 1)
; GFX12-W64-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: wg_st_release_single64
@@ -907,7 +893,7 @@ define amdgpu_kernel void @wg_st_release_single64(ptr addrspace(1) %p, i32 %x) #
; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: GLOBAL_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (store syncscope("workgroup") release (s32) into %ir.2, addrspace 1)
; GFX1250-NEXT: S_ENDPGM 0
store atomic i32 %x, ptr addrspace(1) %p syncscope("workgroup") release, align 4
ret void
@@ -1515,36 +1501,33 @@ define amdgpu_kernel void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) %p, i32
; GFX9: bb.0 (%ir-block.0):
; GFX9-NEXT: liveins: $sgpr4_sgpr5
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX9-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.2, addrspace 1)
; GFX9-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: wg_rmw_xchg_acq_rel_single64
; GFX942: bb.0 (%ir-block.0):
; GFX942-NEXT: liveins: $sgpr4_sgpr5
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.2, addrspace 1)
; GFX942-NEXT: S_ENDPGM 0
;
; GFX10-W32-LABEL: name: wg_rmw_xchg_acq_rel_single64
; GFX10-W32: bb.0 (%ir-block.0):
; GFX10-W32-NEXT: liveins: $sgpr4_sgpr5
; GFX10-W32-NEXT: {{ $}}
- ; GFX10-W32-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W32-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX10-W32-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
; GFX10-W32-NEXT: S_WAITCNT_lds_direct
; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W32-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.2, addrspace 1)
; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
; GFX10-W32-NEXT: S_ENDPGM 0
@@ -1553,11 +1536,10 @@ define amdgpu_kernel void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) %p, i32
; GFX10-W64: bb.0 (%ir-block.0):
; GFX10-W64-NEXT: liveins: $sgpr4_sgpr5
; GFX10-W64-NEXT: {{ $}}
- ; GFX10-W64-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-W64-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX10-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W64-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.2, addrspace 1)
; GFX10-W64-NEXT: S_ENDPGM 0
;
; GFX12-W32-LABEL: name: wg_rmw_xchg_acq_rel_single64
@@ -1571,7 +1553,7 @@ define amdgpu_kernel void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) %p, i32
; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.2, addrspace 1)
; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
; GFX12-W32-NEXT: S_ENDPGM 0
@@ -1583,7 +1565,7 @@ define amdgpu_kernel void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) %p, i32
; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX12-W64-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.2, addrspace 1)
; GFX12-W64-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: wg_rmw_xchg_acq_rel_single64
@@ -1597,7 +1579,7 @@ define amdgpu_kernel void @wg_rmw_xchg_acq_rel_single64(ptr addrspace(1) %p, i32
; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_SWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel (s32) on %ir.2, addrspace 1)
; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
; GFX1250-NEXT: S_ENDPGM 0
%r = atomicrmw xchg ptr addrspace(1) %p, i32 %x syncscope("workgroup") acq_rel
@@ -1613,7 +1595,7 @@ define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1
; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 1)
; GFX9-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
@@ -1624,7 +1606,7 @@ define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1
; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 1)
; GFX942-NEXT: S_ENDPGM 0
;
; GFX10-W32-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
@@ -1638,7 +1620,7 @@ define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1
; GFX10-W32-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
; GFX10-W32-NEXT: S_WAITCNT_lds_direct
; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 1)
; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
; GFX10-W32-NEXT: S_ENDPGM 0
@@ -1651,7 +1633,7 @@ define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1
; GFX10-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 1)
; GFX10-W64-NEXT: S_ENDPGM 0
;
; GFX12-W32-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
@@ -1666,7 +1648,7 @@ define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1
; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 1)
; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
; GFX12-W32-NEXT: S_ENDPGM 0
@@ -1679,7 +1661,7 @@ define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1
; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 1)
; GFX12-W64-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: wg_cmpxchg_acq_rel_monotonic_single64
@@ -1694,7 +1676,7 @@ define amdgpu_kernel void @wg_cmpxchg_acq_rel_monotonic_single64(ptr addrspace(1
; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acq_rel monotonic (s32) on %ir.2, addrspace 1)
; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
; GFX1250-NEXT: S_ENDPGM 0
%r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") acq_rel monotonic
@@ -1712,7 +1694,7 @@ define amdgpu_kernel void @wg_cmpxchg_seq_cst_seq_cst_multi(ptr addrspace(1) %p,
; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.2, addrspace 1)
; GFX9-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
@@ -1725,7 +1707,7 @@ define amdgpu_kernel void @wg_cmpxchg_seq_cst_seq_cst_multi(ptr addrspace(1) %p,
; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.2, addrspace 1)
; GFX942-NEXT: S_ENDPGM 0
;
; GFX10-LABEL: name: wg_cmpxchg_seq_cst_seq_cst_multi
@@ -1739,7 +1721,7 @@ define amdgpu_kernel void @wg_cmpxchg_seq_cst_seq_cst_multi(ptr addrspace(1) %p,
; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
; GFX10-NEXT: S_WAITCNT_lds_direct
; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.2, addrspace 1)
; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
; GFX10-NEXT: BUFFER_GL0_INV implicit $exec
; GFX10-NEXT: S_ENDPGM 0
@@ -1756,7 +1738,7 @@ define amdgpu_kernel void @wg_cmpxchg_seq_cst_seq_cst_multi(ptr addrspace(1) %p,
; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.2, addrspace 1)
; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
; GFX12-W32-NEXT: S_ENDPGM 0
@@ -1774,7 +1756,7 @@ define amdgpu_kernel void @wg_cmpxchg_seq_cst_seq_cst_multi(ptr addrspace(1) %p,
; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.2, addrspace 1)
; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W64-NEXT: GLOBAL_INV 8, implicit $exec
; GFX12-W64-NEXT: S_ENDPGM 0
@@ -1791,7 +1773,7 @@ define amdgpu_kernel void @wg_cmpxchg_seq_cst_seq_cst_multi(ptr addrspace(1) %p,
; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") seq_cst seq_cst (s32) on %ir.2, addrspace 1)
; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
; GFX1250-NEXT: S_ENDPGM 0
%r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") seq_cst seq_cst
@@ -1807,7 +1789,7 @@ define amdgpu_kernel void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1)
; GFX9-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX9-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.2, addrspace 1)
; GFX9-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: wg_cmpxchg_acquire_acquire_single64
@@ -1818,7 +1800,7 @@ define amdgpu_kernel void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1)
; GFX942-NEXT: renamable $vgpr0 = V_MOV_B32_e32 0, implicit $exec
; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr3 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX942-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr0, killed renamable $vgpr2_vgpr3, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.2, addrspace 1)
; GFX942-NEXT: S_ENDPGM 0
;
; GFX10-W32-LABEL: name: wg_cmpxchg_acquire_acquire_single64
@@ -1829,7 +1811,7 @@ define amdgpu_kernel void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1)
; GFX10-W32-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
; GFX10-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
; GFX10-W32-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.2, addrspace 1)
; GFX10-W32-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
; GFX10-W32-NEXT: BUFFER_GL0_INV implicit $exec
; GFX10-W32-NEXT: S_ENDPGM 0
@@ -1842,7 +1824,7 @@ define amdgpu_kernel void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1)
; GFX10-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
; GFX10-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
; GFX10-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX10-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX10-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.2, addrspace 1)
; GFX10-W64-NEXT: S_ENDPGM 0
;
; GFX12-W32-LABEL: name: wg_cmpxchg_acquire_acquire_single64
@@ -1852,7 +1834,7 @@ define amdgpu_kernel void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1)
; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
; GFX12-W32-NEXT: renamable $vgpr2, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 0, killed $sgpr2, implicit $exec, implicit $exec, implicit $exec, implicit $exec
; GFX12-W32-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
- ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W32-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 8, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.2, addrspace 1)
; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W32-NEXT: GLOBAL_INV 8, implicit $exec
; GFX12-W32-NEXT: S_ENDPGM 0
@@ -1865,7 +1847,7 @@ define amdgpu_kernel void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1)
; GFX12-W64-NEXT: renamable $vgpr2 = V_MOV_B32_e32 0, implicit $exec
; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr3, implicit $exec, implicit $exec
; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
- ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX12-W64-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.2, addrspace 1)
; GFX12-W64-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: wg_cmpxchg_acquire_acquire_single64
@@ -1877,7 +1859,7 @@ define amdgpu_kernel void @wg_cmpxchg_acquire_acquire_single64(ptr addrspace(1)
; GFX1250-NEXT: renamable $vgpr2, $vgpr0 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_e96_gfx1250 0, killed $sgpr3, implicit $exec, implicit $exec, implicit $exec, implicit $exec
; GFX1250-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.p.load, addrspace 1)
+ ; GFX1250-NEXT: GLOBAL_ATOMIC_CMPSWAP_SADDR killed renamable $vgpr2, killed renamable $vgpr0_vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec :: (load store syncscope("workgroup") acquire acquire (s32) on %ir.2, addrspace 1)
; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
; GFX1250-NEXT: S_ENDPGM 0
%r = cmpxchg ptr addrspace(1) %p, i32 %cmp, i32 %new syncscope("workgroup") acquire acquire
@@ -2668,42 +2650,40 @@ define amdgpu_kernel void @flat_wg_st_seq_cst_multi(ptr addrspace(0) %p, i32 %x)
; GFX9: bb.0 (%ir-block.0):
; GFX9-NEXT: liveins: $sgpr4_sgpr5
; GFX9-NEXT: {{ $}}
- ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX9-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
- ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX9-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX9-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX9-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
; GFX9-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX9-NEXT: S_WAITCNT_soft .Lgkmcnt_0
; GFX9-NEXT: S_WAITCNT_lds_direct
- ; GFX9-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX9-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.2)
; GFX9-NEXT: S_ENDPGM 0
;
; GFX942-LABEL: name: flat_wg_st_seq_cst_multi
; GFX942: bb.0 (%ir-block.0):
; GFX942-NEXT: liveins: $sgpr4_sgpr5
; GFX942-NEXT: {{ $}}
- ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX942-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX942-NEXT: $vgpr0_vgpr1 = V_MOV_B64_e32 killed $sgpr0_sgpr1, implicit $exec, implicit $exec
+ ; GFX942-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX942-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX942-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
; GFX942-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX942-NEXT: S_WAITCNT_soft .Lgkmcnt_0
; GFX942-NEXT: S_WAITCNT_lds_direct
- ; GFX942-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 1, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX942-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 1, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.2)
; GFX942-NEXT: S_ENDPGM 0
;
; GFX10-LABEL: name: flat_wg_st_seq_cst_multi
; GFX10: bb.0 (%ir-block.0):
; GFX10-NEXT: liveins: $sgpr4_sgpr5
; GFX10-NEXT: {{ $}}
- ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1 = S_LOAD_DWORDX2_IMM_ec renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s64) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX10-NEXT: renamable $sgpr2 = S_LOAD_DWORD_IMM killed renamable $sgpr4_sgpr5, 44, 0 :: (dereferenceable invariant load (s32) from %ir.x.kernarg.offset, addrspace 4)
- ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
- ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX10-NEXT: early-clobber renamable $sgpr0_sgpr1_sgpr2_sgpr3 = S_LOAD_DWORDX4_IMM_ec killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s128) from %ir.p.kernarg.offset, align 4, addrspace 4)
+ ; GFX10-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX10-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
; GFX10-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX10-NEXT: S_WAITCNT_soft .Vmcnt_0_Lgkmcnt_0
; GFX10-NEXT: S_WAITCNT_lds_direct
; GFX10-NEXT: S_WAITCNT_VSCNT_soft undef $sgpr_null, 0
- ; GFX10-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX10-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.2)
; GFX10-NEXT: S_ENDPGM 0
;
; GFX12-W32-LABEL: name: flat_wg_st_seq_cst_multi
@@ -2711,14 +2691,14 @@ define amdgpu_kernel void @flat_wg_st_seq_cst_multi(ptr addrspace(0) %p, i32 %x)
; GFX12-W32-NEXT: liveins: $sgpr4_sgpr5
; GFX12-W32-NEXT: {{ $}}
; GFX12-W32-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX12-W32-NEXT: $vgpr0, $vgpr1 = V_DUAL_MOV_B32_e32_X_MOV_B32_e32_gfx12 killed $sgpr0, killed $sgpr1, implicit $exec, implicit $exec, implicit $exec, implicit $exec, implicit $exec
; GFX12-W32-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX12-W32-NEXT: S_WAIT_BVHCNT_soft 0
; GFX12-W32-NEXT: S_WAIT_SAMPLECNT_soft 0
; GFX12-W32-NEXT: S_WAIT_LOADCNT_soft 0
; GFX12-W32-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W32-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W32-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 8, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX12-W32-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 8, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.2)
; GFX12-W32-NEXT: S_ENDPGM 0
;
; GFX12-W64-LABEL: name: flat_wg_st_seq_cst_multi
@@ -2726,15 +2706,15 @@ define amdgpu_kernel void @flat_wg_st_seq_cst_multi(ptr addrspace(0) %p, i32 %x)
; GFX12-W64-NEXT: liveins: $sgpr4_sgpr5
; GFX12-W64-NEXT: {{ $}}
; GFX12-W64-NEXT: renamable $sgpr0_sgpr1_sgpr2 = S_LOAD_DWORDX3_IMM killed renamable $sgpr4_sgpr5, 36, 0 :: (dereferenceable invariant load (s96) from %ir.p.kernarg.offset, align 4, addrspace 4)
- ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 $sgpr0, implicit $exec, implicit-def $vgpr0_vgpr1, implicit $sgpr0_sgpr1
- ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $sgpr0_sgpr1, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr0 = V_MOV_B32_e32 killed $sgpr0, implicit $exec, implicit $exec
+ ; GFX12-W64-NEXT: $vgpr1 = V_MOV_B32_e32 killed $sgpr1, implicit $exec, implicit $exec
; GFX12-W64-NEXT: $vgpr2 = V_MOV_B32_e32 killed $sgpr2, implicit $exec, implicit $exec
; GFX12-W64-NEXT: S_WAIT_BVHCNT_soft 0
; GFX12-W64-NEXT: S_WAIT_SAMPLECNT_soft 0
; GFX12-W64-NEXT: S_WAIT_LOADCNT_soft 0
; GFX12-W64-NEXT: S_WAIT_STORECNT_soft 0
; GFX12-W64-NEXT: S_WAIT_DSCNT_soft 0
- ; GFX12-W64-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 8, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX12-W64-NEXT: FLAT_STORE_DWORD killed renamable $vgpr0_vgpr1, killed renamable $vgpr2, 0, 8, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.2)
; GFX12-W64-NEXT: S_ENDPGM 0
;
; GFX1250-LABEL: name: flat_wg_st_seq_cst_multi
@@ -2748,7 +2728,7 @@ define amdgpu_kernel void @flat_wg_st_seq_cst_multi(ptr addrspace(0) %p, i32 %x)
; GFX1250-NEXT: S_WAIT_STORECNT_soft 0
; GFX1250-NEXT: S_WAIT_DSCNT_soft 0
; GFX1250-NEXT: S_WAIT_XCNT_soft 0
- ; GFX1250-NEXT: FLAT_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.p.load)
+ ; GFX1250-NEXT: FLAT_STORE_DWORD_SADDR killed renamable $vgpr0, killed renamable $vgpr1, killed renamable $sgpr0_sgpr1, 0, 0, implicit $exec, implicit $flat_scr :: (store syncscope("workgroup") seq_cst (s32) into %ir.2)
; GFX1250-NEXT: S_ENDPGM 0
store atomic i32 %x, ptr addrspace(0) %p syncscope("workgroup") seq_cst, align 4
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/memory_clause.ll b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
index 4d3920a9459dd..59feadb5ba01a 100644
--- a/llvm/test/CodeGen/AMDGPU/memory_clause.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory_clause.ll
@@ -388,14 +388,14 @@ define amdgpu_kernel void @flat_scratch_load(float %a, float %b, <8 x i32> %desc
; GCN-NEXT: s_load_dwordx8 s[8:15], s[4:5], 0x44
; GCN-NEXT: s_addc_u32 s17, s17, 0
; GCN-NEXT: v_mov_b32_e32 v0, 0x40b00000
+; GCN-NEXT: s_brev_b32 s0, 1
; GCN-NEXT: buffer_store_dword v0, off, s[16:19], 0
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: s_brev_b32 s0, 1
-; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s6
; GCN-NEXT: s_mov_b32 s3, 0
; GCN-NEXT: s_mov_b32 s1, s0
; GCN-NEXT: s_mov_b32 s2, s0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v0, s6
; GCN-NEXT: v_mov_b32_e32 v1, s7
; GCN-NEXT: ;;#ASMSTART
; GCN-NEXT: ;;#ASMEND
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 5752bf90cb20f..b83573fd29061 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -591,15 +591,22 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s0
-; GFX942-SDAG-NEXT: v_lshlrev_b16_e32 v3, 8, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-SDAG-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x41e
; GFX942-SDAG-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:1024
-; GFX942-SDAG-NEXT: flat_store_dwordx3 v[0:1], v[4:6] offset:1040
-; GFX942-SDAG-NEXT: flat_store_short v[0:1], v3 offset:1052
-; GFX942-SDAG-NEXT: flat_store_byte v[0:1], v2 offset:1054
+; GFX942-SDAG-NEXT: v_lshlrev_b16_e32 v3, 8, v2
+; GFX942-SDAG-NEXT: v_or_b32_sdwa v8, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[6:7], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x41c
+; GFX942-SDAG-NEXT: flat_store_byte v[6:7], v2
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: s_mov_b64 s[0:1], 0x418
+; GFX942-SDAG-NEXT: flat_store_short v[2:3], v8
+; GFX942-SDAG-NEXT: v_lshl_add_u64 v[2:3], v[0:1], 0, s[0:1]
+; GFX942-SDAG-NEXT: flat_store_dword v[2:3], v4
+; GFX942-SDAG-NEXT: flat_store_dwordx2 v[0:1], v[4:5] offset:1040
; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v61, a13 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v60, a12 ; Reload Reuse
; GFX942-SDAG-NEXT: v_accvgpr_read_b32 v59, a11 ; Reload Reuse
@@ -662,19 +669,25 @@ define void @memset_p0_sz1055_align_4_varsetval(ptr addrspace(0) align 4 %dst, i
; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v3, 8, v3
; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v3
; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; GFX942-GISEL-NEXT: v_or3_b32 v6, v4, v5, v3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 8
-; GFX942-GISEL-NEXT: v_lshlrev_b16_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX942-GISEL-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v3
-; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v4
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v6
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v6
+; GFX942-GISEL-NEXT: v_or3_b32 v4, v4, v5, v3
; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-GISEL-NEXT: flat_store_dwordx4 v[0:1], v[6:9] offset:1024
-; GFX942-GISEL-NEXT: flat_store_dwordx3 v[0:1], v[4:6] offset:1040
-; GFX942-GISEL-NEXT: flat_store_short v[0:1], v3 offset:1052
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-GISEL-NEXT: flat_store_dwordx4 v[0:1], v[4:7] offset:1024
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1040
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1041
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1042
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1043
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1044
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1045
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1046
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1047
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1048
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1049
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1050
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1051
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1052
+; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1053
; GFX942-GISEL-NEXT: flat_store_byte v[0:1], v2 offset:1054
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -911,13 +924,14 @@ define void @memset_p1_sz1055_align_4_varsetval(ptr addrspace(1) align 4 %dst, i
; GFX942-SDAG-NEXT: v_perm_b32 v4, v2, v2, s0
; GFX942-SDAG-NEXT: v_lshlrev_b16_e32 v3, 8, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v4
+; GFX942-SDAG-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX942-SDAG-NEXT: v_mov_b32_e32 v6, v4
; GFX942-SDAG-NEXT: v_mov_b32_e32 v7, v4
-; GFX942-SDAG-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-SDAG-NEXT: global_store_dwordx4 v[0:1], v[4:7], off offset:1024
-; GFX942-SDAG-NEXT: global_store_dwordx3 v[0:1], v[4:6], off offset:1040
-; GFX942-SDAG-NEXT: global_store_short v[0:1], v3, off offset:1052
; GFX942-SDAG-NEXT: global_store_byte v[0:1], v2, off offset:1054
+; GFX942-SDAG-NEXT: global_store_short v[0:1], v3, off offset:1052
+; GFX942-SDAG-NEXT: v_mov_b64_e32 v[2:3], v[4:5]
+; GFX942-SDAG-NEXT: global_store_dwordx4 v[0:1], v[4:7], off offset:1024
+; GFX942-SDAG-NEXT: global_store_dwordx3 v[0:1], v[2:4], off offset:1040
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -966,19 +980,25 @@ define void @memset_p1_sz1055_align_4_varsetval(ptr addrspace(1) align 4 %dst, i
; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v3, 8, v3
; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v5, 16, v3
; GFX942-GISEL-NEXT: v_lshlrev_b32_e32 v3, 24, v3
-; GFX942-GISEL-NEXT: v_or3_b32 v6, v4, v5, v3
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 8
-; GFX942-GISEL-NEXT: v_lshlrev_b16_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX942-GISEL-NEXT: v_or_b32_sdwa v3, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v3
-; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v4
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v6
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v8, v6
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v9, v6
+; GFX942-GISEL-NEXT: v_or3_b32 v4, v4, v5, v3
; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[6:9], off offset:1024
-; GFX942-GISEL-NEXT: global_store_dwordx3 v[0:1], v[4:6], off offset:1040
-; GFX942-GISEL-NEXT: global_store_short v[0:1], v3, off offset:1052
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v6, v4
+; GFX942-GISEL-NEXT: v_mov_b32_e32 v7, v4
+; GFX942-GISEL-NEXT: global_store_dwordx4 v[0:1], v[4:7], off offset:1024
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1040
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1041
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1042
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1043
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1044
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1045
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1046
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1047
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1048
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1049
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1050
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1051
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1052
+; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1053
; GFX942-GISEL-NEXT: global_store_byte v[0:1], v2, off offset:1054
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
@@ -1193,8 +1213,8 @@ define void @memset_p3_sz1055_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-SDAG-NEXT: ds_write_b32 v0, v4 offset:1048
; GFX942-SDAG-NEXT: v_lshlrev_b16_e32 v2, 8, v1
; GFX942-SDAG-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-SDAG-NEXT: ds_write_b16 v0, v2 offset:1052
; GFX942-SDAG-NEXT: ds_write_b8 v0, v1 offset:1054
+; GFX942-SDAG-NEXT: ds_write_b16 v0, v2 offset:1052
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1249,13 +1269,16 @@ define void @memset_p3_sz1055_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 8
; GFX942-GISEL-NEXT: v_lshlrev_b16_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
; GFX942-GISEL-NEXT: v_or_b32_sdwa v3, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v3
-; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v4
+; GFX942-GISEL-NEXT: s_mov_b32 s0, 0xffff
+; GFX942-GISEL-NEXT: v_and_b32_e32 v3, 0xffff, v3
+; GFX942-GISEL-NEXT: v_and_b32_sdwa v1, s0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v3, 16, v3
; GFX942-GISEL-NEXT: v_add_u32_e32 v5, 0x410, v0
+; GFX942-GISEL-NEXT: v_lshl_or_b32 v1, v1, 16, v3
; GFX942-GISEL-NEXT: ds_write2_b32 v5, v4, v4 offset1:1
; GFX942-GISEL-NEXT: ds_write_b32 v0, v2 offset:1048
-; GFX942-GISEL-NEXT: ds_write_b16 v0, v3 offset:1052
-; GFX942-GISEL-NEXT: ds_write_b8 v0, v1 offset:1054
+; GFX942-GISEL-NEXT: ds_write_b16 v0, v1 offset:1052
+; GFX942-GISEL-NEXT: ds_write_b8_d16_hi v0, v1 offset:1054
; GFX942-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -1460,12 +1483,14 @@ define void @memset_p5_sz1055_align_4_varsetval(ptr addrspace(5) align 4 %dst, i
; GFX942-SDAG-NEXT: v_mov_b32_e32 v4, v2
; GFX942-SDAG-NEXT: v_mov_b32_e32 v5, v2
; GFX942-SDAG-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:1024
-; GFX942-SDAG-NEXT: scratch_store_dwordx2 v0, v[2:3], off offset:1040
-; GFX942-SDAG-NEXT: scratch_store_dword v0, v2, off offset:1048
-; GFX942-SDAG-NEXT: v_lshlrev_b16_e32 v2, 8, v1
-; GFX942-SDAG-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-SDAG-NEXT: scratch_store_short v0, v2, off offset:1052
+; GFX942-SDAG-NEXT: s_nop 1
+; GFX942-SDAG-NEXT: v_lshlrev_b16_e32 v3, 8, v1
+; GFX942-SDAG-NEXT: v_or_b32_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-SDAG-NEXT: v_mov_b32_e32 v3, v2
; GFX942-SDAG-NEXT: scratch_store_byte v0, v1, off offset:1054
+; GFX942-SDAG-NEXT: scratch_store_short v0, v4, off offset:1052
+; GFX942-SDAG-NEXT: scratch_store_dword v0, v2, off offset:1048
+; GFX942-SDAG-NEXT: scratch_store_dwordx2 v0, v[2:3], off offset:1040
; GFX942-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX942-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -1517,16 +1542,20 @@ define void @memset_p5_sz1055_align_4_varsetval(ptr addrspace(5) align 4 %dst, i
; GFX942-GISEL-NEXT: v_mov_b32_e32 v4, v2
; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v2
; GFX942-GISEL-NEXT: scratch_store_dwordx4 v0, v[2:5], off offset:1024
-; GFX942-GISEL-NEXT: s_nop 1
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v3, 8
-; GFX942-GISEL-NEXT: v_lshlrev_b16_sdwa v3, v3, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
-; GFX942-GISEL-NEXT: v_or_b32_sdwa v3, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
-; GFX942-GISEL-NEXT: v_and_b32_e32 v4, 0xffff, v3
-; GFX942-GISEL-NEXT: v_lshl_or_b32 v4, v4, 16, v4
-; GFX942-GISEL-NEXT: v_mov_b32_e32 v5, v4
-; GFX942-GISEL-NEXT: scratch_store_dwordx2 v0, v[4:5], off offset:1040
-; GFX942-GISEL-NEXT: scratch_store_dword v0, v2, off offset:1048
-; GFX942-GISEL-NEXT: scratch_store_short v0, v3, off offset:1052
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1040
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1041
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1042
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1043
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1044
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1045
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1046
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1047
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1048
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1049
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1050
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1051
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1052
+; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1053
; GFX942-GISEL-NEXT: scratch_store_byte v0, v1, off offset:1054
; GFX942-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX942-GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
index 263298d1cd1c8..892802d80776f 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-cd-select.ll
@@ -745,24 +745,17 @@ bb:
define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call_multi_bb(ptr addrspace(1) %arg, i1 %c0) #1 {
; GFX908-LABEL: test_mfma_f32_32x32x1f32_call_multi_bb:
; GFX908: ; %bb.0: ; %bb1
-; GFX908-NEXT: s_mov_b32 s52, SCRATCH_RSRC_DWORD0
-; GFX908-NEXT: s_mov_b32 s53, SCRATCH_RSRC_DWORD1
-; GFX908-NEXT: s_mov_b32 s54, -1
-; GFX908-NEXT: s_mov_b32 s55, 0xe00000
-; GFX908-NEXT: s_add_u32 s52, s52, s11
-; GFX908-NEXT: s_mov_b32 s14, s10
-; GFX908-NEXT: s_mov_b32 s12, s8
-; GFX908-NEXT: s_mov_b64 s[10:11], s[6:7]
-; GFX908-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX908-NEXT: s_load_dword s8, s[4:5], 0x2c
+; GFX908-NEXT: s_load_dwordx4 s[52:55], s[4:5], 0x24
; GFX908-NEXT: v_mov_b32_e32 v6, 1.0
; GFX908-NEXT: v_mov_b32_e32 v7, 0
-; GFX908-NEXT: s_addc_u32 s53, s53, 0
+; GFX908-NEXT: s_mov_b32 s64, SCRATCH_RSRC_DWORD0
+; GFX908-NEXT: s_mov_b32 s65, SCRATCH_RSRC_DWORD1
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: s_load_dwordx16 s[36:51], s[6:7], 0x0
-; GFX908-NEXT: s_load_dwordx16 s[16:31], s[6:7], 0x40
-; GFX908-NEXT: s_bitcmp0_b32 s8, 0
-; GFX908-NEXT: s_mov_b32 s32, 0
+; GFX908-NEXT: s_load_dwordx16 s[36:51], s[52:53], 0x0
+; GFX908-NEXT: s_load_dwordx16 s[16:31], s[52:53], 0x40
+; GFX908-NEXT: s_mov_b32 s66, -1
+; GFX908-NEXT: s_mov_b32 s67, 0xe00000
+; GFX908-NEXT: s_add_u32 s64, s64, s11
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: v_mov_b32_e32 v3, s36
; GFX908-NEXT: v_mov_b32_e32 v4, s37
@@ -829,67 +822,70 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call_multi_bb(ptr addrspace(
; GFX908-NEXT: v_accvgpr_write_b32 a30, v4
; GFX908-NEXT: v_accvgpr_write_b32 a31, v5
; GFX908-NEXT: v_mov_b32_e32 v3, 2.0
-; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: s_addc_u32 s65, s65, 0
+; GFX908-NEXT: s_bitcmp0_b32 s54, 0
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v6, v3, a[0:31] cbsz:1 abid:2 blgp:3
+; GFX908-NEXT: s_mov_b32 s32, 0
; GFX908-NEXT: s_nop 15
-; GFX908-NEXT: s_nop 1
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: v_accvgpr_read_b32 v6, a27
; GFX908-NEXT: v_accvgpr_read_b32 v5, a26
; GFX908-NEXT: v_accvgpr_read_b32 v4, a25
; GFX908-NEXT: v_accvgpr_read_b32 v3, a24
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:96
+; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[52:53] offset:96
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: v_accvgpr_read_b32 v6, a31
; GFX908-NEXT: v_accvgpr_read_b32 v5, a30
; GFX908-NEXT: v_accvgpr_read_b32 v4, a29
; GFX908-NEXT: v_accvgpr_read_b32 v3, a28
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:112
+; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[52:53] offset:112
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: v_accvgpr_read_b32 v6, a19
; GFX908-NEXT: v_accvgpr_read_b32 v5, a18
; GFX908-NEXT: v_accvgpr_read_b32 v4, a17
; GFX908-NEXT: v_accvgpr_read_b32 v3, a16
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:64
+; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[52:53] offset:64
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: v_accvgpr_read_b32 v6, a23
; GFX908-NEXT: v_accvgpr_read_b32 v5, a22
; GFX908-NEXT: v_accvgpr_read_b32 v4, a21
; GFX908-NEXT: v_accvgpr_read_b32 v3, a20
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:80
+; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[52:53] offset:80
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: v_accvgpr_read_b32 v6, a11
; GFX908-NEXT: v_accvgpr_read_b32 v5, a10
; GFX908-NEXT: v_accvgpr_read_b32 v4, a9
; GFX908-NEXT: v_accvgpr_read_b32 v3, a8
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:32
+; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[52:53] offset:32
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: v_accvgpr_read_b32 v6, a15
; GFX908-NEXT: v_accvgpr_read_b32 v5, a14
; GFX908-NEXT: v_accvgpr_read_b32 v4, a13
; GFX908-NEXT: v_accvgpr_read_b32 v3, a12
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:48
+; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[52:53] offset:48
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: v_accvgpr_read_b32 v6, a3
; GFX908-NEXT: v_accvgpr_read_b32 v5, a2
; GFX908-NEXT: v_accvgpr_read_b32 v4, a1
; GFX908-NEXT: v_accvgpr_read_b32 v3, a0
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7]
+; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[52:53]
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: v_accvgpr_read_b32 v6, a7
; GFX908-NEXT: v_accvgpr_read_b32 v5, a6
; GFX908-NEXT: v_accvgpr_read_b32 v4, a5
; GFX908-NEXT: v_accvgpr_read_b32 v3, a4
; GFX908-NEXT: s_nop 1
-; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[6:7] offset:16
+; GFX908-NEXT: global_store_dwordx4 v7, v[3:6], s[52:53] offset:16
; GFX908-NEXT: s_cbranch_scc1 .LBB6_2
; GFX908-NEXT: ; %bb.1: ; %bb2
+; GFX908-NEXT: s_mov_b32 s12, s8
; GFX908-NEXT: s_add_u32 s8, s4, 48
; GFX908-NEXT: s_mov_b32 s13, s9
; GFX908-NEXT: s_addc_u32 s9, s5, 0
@@ -897,13 +893,15 @@ define amdgpu_kernel void @test_mfma_f32_32x32x1f32_call_multi_bb(ptr addrspace(
; GFX908-NEXT: s_add_u32 s4, s4, foo at gotpcrel32@lo+4
; GFX908-NEXT: s_addc_u32 s5, s5, foo at gotpcrel32@hi+12
; GFX908-NEXT: s_load_dwordx2 s[16:17], s[4:5], 0x0
+; GFX908-NEXT: s_mov_b32 s14, s10
+; GFX908-NEXT: s_mov_b64 s[10:11], s[6:7]
; GFX908-NEXT: v_lshlrev_b32_e32 v2, 20, v2
; GFX908-NEXT: v_lshlrev_b32_e32 v1, 10, v1
; GFX908-NEXT: s_mov_b64 s[4:5], s[0:1]
; GFX908-NEXT: s_mov_b64 s[6:7], s[2:3]
-; GFX908-NEXT: s_mov_b64 s[0:1], s[52:53]
+; GFX908-NEXT: s_mov_b64 s[0:1], s[64:65]
; GFX908-NEXT: v_or3_b32 v31, v0, v1, v2
-; GFX908-NEXT: s_mov_b64 s[2:3], s[54:55]
+; GFX908-NEXT: s_mov_b64 s[2:3], s[66:67]
; GFX908-NEXT: ; implicit-def: $sgpr15
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: s_swappc_b64 s[30:31], s[16:17]
diff --git a/llvm/test/CodeGen/AMDGPU/mfma-loop.ll b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
index 059527cea7c01..ffe2d5e99d601 100644
--- a/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
+++ b/llvm/test/CodeGen/AMDGPU/mfma-loop.ll
@@ -1213,11 +1213,11 @@ exit:
define amdgpu_kernel void @test_mfma_loop_sgpr_init(ptr addrspace(1) %arg, float %init) #0 {
; GFX908-LABEL: test_mfma_loop_sgpr_init:
; GFX908: ; %bb.0: ; %entry
-; GFX908-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX908-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX908-NEXT: v_mov_b32_e32 v1, 1.0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v0, s0
-; GFX908-NEXT: s_mov_b32 s0, 16
+; GFX908-NEXT: v_mov_b32_e32 v0, s2
+; GFX908-NEXT: s_mov_b32 s2, 16
; GFX908-NEXT: s_nop 0
; GFX908-NEXT: v_accvgpr_write_b32 a31, v0
; GFX908-NEXT: v_accvgpr_write_b32 a30, v0
@@ -1256,12 +1256,11 @@ define amdgpu_kernel void @test_mfma_loop_sgpr_init(ptr addrspace(1) %arg, float
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v0, a[0:31]
-; GFX908-NEXT: s_add_i32 s0, s0, -1
-; GFX908-NEXT: s_cmp_lg_u32 s0, 0
+; GFX908-NEXT: s_add_i32 s2, s2, -1
+; GFX908-NEXT: s_cmp_lg_u32 s2, 0
; GFX908-NEXT: s_cbranch_scc1 .LBB5_1
; GFX908-NEXT: ; %bb.2: ; %exit
-; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX908-NEXT: s_nop 13
+; GFX908-NEXT: s_nop 14
; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
; GFX908-NEXT: v_accvgpr_read_b32 v28, a28
; GFX908-NEXT: v_accvgpr_read_b32 v29, a29
@@ -1295,8 +1294,8 @@ define amdgpu_kernel void @test_mfma_loop_sgpr_init(ptr addrspace(1) %arg, float
; GFX908-NEXT: v_accvgpr_read_b32 v25, a25
; GFX908-NEXT: v_accvgpr_read_b32 v26, a26
; GFX908-NEXT: v_accvgpr_read_b32 v27, a27
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
@@ -1308,55 +1307,53 @@ define amdgpu_kernel void @test_mfma_loop_sgpr_init(ptr addrspace(1) %arg, float
;
; GFX90A-LABEL: test_mfma_loop_sgpr_init:
; GFX90A: ; %bb.0: ; %entry
-; GFX90A-NEXT: s_load_dword s1, s[4:5], 0x2c
-; GFX90A-NEXT: s_mov_b32 s0, 16
+; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b32 s3, 16
; GFX90A-NEXT: v_mov_b32_e32 v32, 2.0
; GFX90A-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v0, s1
-; GFX90A-NEXT: v_mov_b32_e32 v1, s1
-; GFX90A-NEXT: v_mov_b32_e32 v2, s1
-; GFX90A-NEXT: v_mov_b32_e32 v3, s1
-; GFX90A-NEXT: v_mov_b32_e32 v4, s1
-; GFX90A-NEXT: v_mov_b32_e32 v5, s1
-; GFX90A-NEXT: v_mov_b32_e32 v6, s1
-; GFX90A-NEXT: v_mov_b32_e32 v7, s1
-; GFX90A-NEXT: v_mov_b32_e32 v8, s1
-; GFX90A-NEXT: v_mov_b32_e32 v9, s1
-; GFX90A-NEXT: v_mov_b32_e32 v10, s1
-; GFX90A-NEXT: v_mov_b32_e32 v11, s1
-; GFX90A-NEXT: v_mov_b32_e32 v12, s1
-; GFX90A-NEXT: v_mov_b32_e32 v13, s1
-; GFX90A-NEXT: v_mov_b32_e32 v14, s1
-; GFX90A-NEXT: v_mov_b32_e32 v15, s1
-; GFX90A-NEXT: v_mov_b32_e32 v16, s1
-; GFX90A-NEXT: v_mov_b32_e32 v17, s1
-; GFX90A-NEXT: v_mov_b32_e32 v18, s1
-; GFX90A-NEXT: v_mov_b32_e32 v19, s1
-; GFX90A-NEXT: v_mov_b32_e32 v20, s1
-; GFX90A-NEXT: v_mov_b32_e32 v21, s1
-; GFX90A-NEXT: v_mov_b32_e32 v22, s1
-; GFX90A-NEXT: v_mov_b32_e32 v23, s1
-; GFX90A-NEXT: v_mov_b32_e32 v24, s1
-; GFX90A-NEXT: v_mov_b32_e32 v25, s1
-; GFX90A-NEXT: v_mov_b32_e32 v26, s1
-; GFX90A-NEXT: v_mov_b32_e32 v27, s1
-; GFX90A-NEXT: v_mov_b32_e32 v28, s1
-; GFX90A-NEXT: v_mov_b32_e32 v29, s1
-; GFX90A-NEXT: v_mov_b32_e32 v30, s1
-; GFX90A-NEXT: v_mov_b32_e32 v31, s1
+; GFX90A-NEXT: v_mov_b32_e32 v0, s2
+; GFX90A-NEXT: v_mov_b32_e32 v1, s2
+; GFX90A-NEXT: v_mov_b32_e32 v2, s2
+; GFX90A-NEXT: v_mov_b32_e32 v3, s2
+; GFX90A-NEXT: v_mov_b32_e32 v4, s2
+; GFX90A-NEXT: v_mov_b32_e32 v5, s2
+; GFX90A-NEXT: v_mov_b32_e32 v6, s2
+; GFX90A-NEXT: v_mov_b32_e32 v7, s2
+; GFX90A-NEXT: v_mov_b32_e32 v8, s2
+; GFX90A-NEXT: v_mov_b32_e32 v9, s2
+; GFX90A-NEXT: v_mov_b32_e32 v10, s2
+; GFX90A-NEXT: v_mov_b32_e32 v11, s2
+; GFX90A-NEXT: v_mov_b32_e32 v12, s2
+; GFX90A-NEXT: v_mov_b32_e32 v13, s2
+; GFX90A-NEXT: v_mov_b32_e32 v14, s2
+; GFX90A-NEXT: v_mov_b32_e32 v15, s2
+; GFX90A-NEXT: v_mov_b32_e32 v16, s2
+; GFX90A-NEXT: v_mov_b32_e32 v17, s2
+; GFX90A-NEXT: v_mov_b32_e32 v18, s2
+; GFX90A-NEXT: v_mov_b32_e32 v19, s2
+; GFX90A-NEXT: v_mov_b32_e32 v20, s2
+; GFX90A-NEXT: v_mov_b32_e32 v21, s2
+; GFX90A-NEXT: v_mov_b32_e32 v22, s2
+; GFX90A-NEXT: v_mov_b32_e32 v23, s2
+; GFX90A-NEXT: v_mov_b32_e32 v24, s2
+; GFX90A-NEXT: v_mov_b32_e32 v25, s2
+; GFX90A-NEXT: v_mov_b32_e32 v26, s2
+; GFX90A-NEXT: v_mov_b32_e32 v27, s2
+; GFX90A-NEXT: v_mov_b32_e32 v28, s2
+; GFX90A-NEXT: v_mov_b32_e32 v29, s2
+; GFX90A-NEXT: v_mov_b32_e32 v30, s2
+; GFX90A-NEXT: v_mov_b32_e32 v31, s2
; GFX90A-NEXT: .LBB5_1: ; %for.cond.preheader
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_nop 1
; GFX90A-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v33, v32, v[0:31]
-; GFX90A-NEXT: s_add_i32 s0, s0, -1
-; GFX90A-NEXT: s_cmp_lg_u32 s0, 0
+; GFX90A-NEXT: s_add_i32 s3, s3, -1
+; GFX90A-NEXT: s_cmp_lg_u32 s3, 0
; GFX90A-NEXT: s_cbranch_scc1 .LBB5_1
; GFX90A-NEXT: ; %bb.2: ; %exit
-; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX90A-NEXT: v_mov_b32_e32 v32, 0
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_nop 12
+; GFX90A-NEXT: s_nop 14
; GFX90A-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
; GFX90A-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
; GFX90A-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
@@ -1369,55 +1366,53 @@ define amdgpu_kernel void @test_mfma_loop_sgpr_init(ptr addrspace(1) %arg, float
;
; GFX942-LABEL: test_mfma_loop_sgpr_init:
; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_load_dword s1, s[4:5], 0x2c
-; GFX942-NEXT: s_mov_b32 s0, 16
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s3, 16
; GFX942-NEXT: v_mov_b32_e32 v32, 2.0
; GFX942-NEXT: v_mov_b32_e32 v33, 1.0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v0, s1
-; GFX942-NEXT: v_mov_b32_e32 v1, s1
-; GFX942-NEXT: v_mov_b32_e32 v2, s1
-; GFX942-NEXT: v_mov_b32_e32 v3, s1
-; GFX942-NEXT: v_mov_b32_e32 v4, s1
-; GFX942-NEXT: v_mov_b32_e32 v5, s1
-; GFX942-NEXT: v_mov_b32_e32 v6, s1
-; GFX942-NEXT: v_mov_b32_e32 v7, s1
-; GFX942-NEXT: v_mov_b32_e32 v8, s1
-; GFX942-NEXT: v_mov_b32_e32 v9, s1
-; GFX942-NEXT: v_mov_b32_e32 v10, s1
-; GFX942-NEXT: v_mov_b32_e32 v11, s1
-; GFX942-NEXT: v_mov_b32_e32 v12, s1
-; GFX942-NEXT: v_mov_b32_e32 v13, s1
-; GFX942-NEXT: v_mov_b32_e32 v14, s1
-; GFX942-NEXT: v_mov_b32_e32 v15, s1
-; GFX942-NEXT: v_mov_b32_e32 v16, s1
-; GFX942-NEXT: v_mov_b32_e32 v17, s1
-; GFX942-NEXT: v_mov_b32_e32 v18, s1
-; GFX942-NEXT: v_mov_b32_e32 v19, s1
-; GFX942-NEXT: v_mov_b32_e32 v20, s1
-; GFX942-NEXT: v_mov_b32_e32 v21, s1
-; GFX942-NEXT: v_mov_b32_e32 v22, s1
-; GFX942-NEXT: v_mov_b32_e32 v23, s1
-; GFX942-NEXT: v_mov_b32_e32 v24, s1
-; GFX942-NEXT: v_mov_b32_e32 v25, s1
-; GFX942-NEXT: v_mov_b32_e32 v26, s1
-; GFX942-NEXT: v_mov_b32_e32 v27, s1
-; GFX942-NEXT: v_mov_b32_e32 v28, s1
-; GFX942-NEXT: v_mov_b32_e32 v29, s1
-; GFX942-NEXT: v_mov_b32_e32 v30, s1
-; GFX942-NEXT: v_mov_b32_e32 v31, s1
+; GFX942-NEXT: v_mov_b32_e32 v0, s2
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NEXT: v_mov_b32_e32 v2, s2
+; GFX942-NEXT: v_mov_b32_e32 v3, s2
+; GFX942-NEXT: v_mov_b32_e32 v4, s2
+; GFX942-NEXT: v_mov_b32_e32 v5, s2
+; GFX942-NEXT: v_mov_b32_e32 v6, s2
+; GFX942-NEXT: v_mov_b32_e32 v7, s2
+; GFX942-NEXT: v_mov_b32_e32 v8, s2
+; GFX942-NEXT: v_mov_b32_e32 v9, s2
+; GFX942-NEXT: v_mov_b32_e32 v10, s2
+; GFX942-NEXT: v_mov_b32_e32 v11, s2
+; GFX942-NEXT: v_mov_b32_e32 v12, s2
+; GFX942-NEXT: v_mov_b32_e32 v13, s2
+; GFX942-NEXT: v_mov_b32_e32 v14, s2
+; GFX942-NEXT: v_mov_b32_e32 v15, s2
+; GFX942-NEXT: v_mov_b32_e32 v16, s2
+; GFX942-NEXT: v_mov_b32_e32 v17, s2
+; GFX942-NEXT: v_mov_b32_e32 v18, s2
+; GFX942-NEXT: v_mov_b32_e32 v19, s2
+; GFX942-NEXT: v_mov_b32_e32 v20, s2
+; GFX942-NEXT: v_mov_b32_e32 v21, s2
+; GFX942-NEXT: v_mov_b32_e32 v22, s2
+; GFX942-NEXT: v_mov_b32_e32 v23, s2
+; GFX942-NEXT: v_mov_b32_e32 v24, s2
+; GFX942-NEXT: v_mov_b32_e32 v25, s2
+; GFX942-NEXT: v_mov_b32_e32 v26, s2
+; GFX942-NEXT: v_mov_b32_e32 v27, s2
+; GFX942-NEXT: v_mov_b32_e32 v28, s2
+; GFX942-NEXT: v_mov_b32_e32 v29, s2
+; GFX942-NEXT: v_mov_b32_e32 v30, s2
+; GFX942-NEXT: v_mov_b32_e32 v31, s2
; GFX942-NEXT: .LBB5_1: ; %for.cond.preheader
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_nop 1
; GFX942-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v32, v[0:31]
-; GFX942-NEXT: s_add_i32 s0, s0, -1
-; GFX942-NEXT: s_cmp_lg_u32 s0, 0
+; GFX942-NEXT: s_add_i32 s3, s3, -1
+; GFX942-NEXT: s_cmp_lg_u32 s3, 0
; GFX942-NEXT: s_cbranch_scc1 .LBB5_1
; GFX942-NEXT: ; %bb.2: ; %exit
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_nop 11
+; GFX942-NEXT: s_nop 13
; GFX942-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
; GFX942-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
; GFX942-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
@@ -1479,13 +1474,13 @@ exit:
define amdgpu_kernel void @test_mfma_loop_mixed_init(ptr addrspace(1) %arg, float %x) #0 {
; GFX908-LABEL: test_mfma_loop_mixed_init:
; GFX908: ; %bb.0: ; %entry
-; GFX908-NEXT: s_load_dword s1, s[4:5], 0x2c
+; GFX908-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX908-NEXT: v_accvgpr_write_b32 a0, v0
; GFX908-NEXT: v_accvgpr_write_b32 a31, 0
; GFX908-NEXT: v_accvgpr_write_b32 a30, 0
; GFX908-NEXT: v_accvgpr_write_b32 a29, 0
; GFX908-NEXT: s_waitcnt lgkmcnt(0)
-; GFX908-NEXT: v_mov_b32_e32 v0, s1
+; GFX908-NEXT: v_mov_b32_e32 v0, s2
; GFX908-NEXT: v_accvgpr_write_b32 a28, 0
; GFX908-NEXT: v_accvgpr_write_b32 a27, 0
; GFX908-NEXT: v_accvgpr_write_b32 a26, 0
@@ -1514,19 +1509,18 @@ define amdgpu_kernel void @test_mfma_loop_mixed_init(ptr addrspace(1) %arg, floa
; GFX908-NEXT: v_accvgpr_write_b32 a3, 0
; GFX908-NEXT: v_accvgpr_write_b32 a2, 0
; GFX908-NEXT: v_accvgpr_write_b32 a1, v0
-; GFX908-NEXT: s_mov_b32 s0, 16
+; GFX908-NEXT: s_mov_b32 s3, 16
; GFX908-NEXT: v_mov_b32_e32 v0, 2.0
; GFX908-NEXT: v_mov_b32_e32 v1, 1.0
; GFX908-NEXT: .LBB6_1: ; %for.cond.preheader
; GFX908-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX908-NEXT: s_nop 1
; GFX908-NEXT: v_mfma_f32_32x32x1f32 a[0:31], v1, v0, a[0:31]
-; GFX908-NEXT: s_add_i32 s0, s0, -1
-; GFX908-NEXT: s_cmp_lg_u32 s0, 0
+; GFX908-NEXT: s_add_i32 s3, s3, -1
+; GFX908-NEXT: s_cmp_lg_u32 s3, 0
; GFX908-NEXT: s_cbranch_scc1 .LBB6_1
; GFX908-NEXT: ; %bb.2: ; %exit
-; GFX908-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX908-NEXT: s_nop 13
+; GFX908-NEXT: s_nop 14
; GFX908-NEXT: v_accvgpr_read_b32 v0, a0
; GFX908-NEXT: v_accvgpr_read_b32 v28, a28
; GFX908-NEXT: v_accvgpr_read_b32 v29, a29
@@ -1560,8 +1554,8 @@ define amdgpu_kernel void @test_mfma_loop_mixed_init(ptr addrspace(1) %arg, floa
; GFX908-NEXT: v_accvgpr_read_b32 v25, a25
; GFX908-NEXT: v_accvgpr_read_b32 v26, a26
; GFX908-NEXT: v_accvgpr_read_b32 v27, a27
-; GFX908-NEXT: s_waitcnt lgkmcnt(0)
; GFX908-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
+; GFX908-NEXT: s_nop 0
; GFX908-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
; GFX908-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
; GFX908-NEXT: global_store_dwordx4 v32, v[16:19], s[0:1] offset:64
@@ -1573,13 +1567,13 @@ define amdgpu_kernel void @test_mfma_loop_mixed_init(ptr addrspace(1) %arg, floa
;
; GFX90A-LABEL: test_mfma_loop_mixed_init:
; GFX90A: ; %bb.0: ; %entry
-; GFX90A-NEXT: s_load_dword s1, s[4:5], 0x2c
+; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX90A-NEXT: v_mov_b32_e32 v2, 0
-; GFX90A-NEXT: s_mov_b32 s0, 16
+; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NEXT: s_mov_b32 s3, 16
; GFX90A-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX90A-NEXT: v_mov_b32_e32 v32, 2.0
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: v_mov_b32_e32 v1, s1
+; GFX90A-NEXT: v_mov_b32_e32 v1, s2
; GFX90A-NEXT: v_mov_b32_e32 v33, 1.0
; GFX90A-NEXT: v_mov_b32_e32 v3, v2
; GFX90A-NEXT: v_mov_b32_e32 v4, v2
@@ -1614,14 +1608,12 @@ define amdgpu_kernel void @test_mfma_loop_mixed_init(ptr addrspace(1) %arg, floa
; GFX90A-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX90A-NEXT: s_nop 1
; GFX90A-NEXT: v_mfma_f32_32x32x1f32 v[0:31], v33, v32, v[0:31]
-; GFX90A-NEXT: s_add_i32 s0, s0, -1
-; GFX90A-NEXT: s_cmp_lg_u32 s0, 0
+; GFX90A-NEXT: s_add_i32 s3, s3, -1
+; GFX90A-NEXT: s_cmp_lg_u32 s3, 0
; GFX90A-NEXT: s_cbranch_scc1 .LBB6_1
; GFX90A-NEXT: ; %bb.2: ; %exit
-; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX90A-NEXT: v_mov_b32_e32 v32, 0
-; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_nop 12
+; GFX90A-NEXT: s_nop 14
; GFX90A-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
; GFX90A-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
; GFX90A-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
@@ -1634,13 +1626,13 @@ define amdgpu_kernel void @test_mfma_loop_mixed_init(ptr addrspace(1) %arg, floa
;
; GFX942-LABEL: test_mfma_loop_mixed_init:
; GFX942: ; %bb.0: ; %entry
-; GFX942-NEXT: s_load_dword s1, s[4:5], 0x2c
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX942-NEXT: v_mov_b32_e32 v2, 0
-; GFX942-NEXT: s_mov_b32 s0, 16
+; GFX942-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NEXT: s_mov_b32 s3, 16
; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX942-NEXT: v_mov_b32_e32 v32, 2.0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: v_mov_b32_e32 v1, s1
+; GFX942-NEXT: v_mov_b32_e32 v1, s2
; GFX942-NEXT: v_mov_b32_e32 v33, 1.0
; GFX942-NEXT: v_mov_b32_e32 v3, v2
; GFX942-NEXT: v_mov_b32_e32 v4, v2
@@ -1675,14 +1667,12 @@ define amdgpu_kernel void @test_mfma_loop_mixed_init(ptr addrspace(1) %arg, floa
; GFX942-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-NEXT: s_nop 1
; GFX942-NEXT: v_mfma_f32_32x32x1_2b_f32 v[0:31], v33, v32, v[0:31]
-; GFX942-NEXT: s_add_i32 s0, s0, -1
-; GFX942-NEXT: s_cmp_lg_u32 s0, 0
+; GFX942-NEXT: s_add_i32 s3, s3, -1
+; GFX942-NEXT: s_cmp_lg_u32 s3, 0
; GFX942-NEXT: s_cbranch_scc1 .LBB6_1
; GFX942-NEXT: ; %bb.2: ; %exit
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX942-NEXT: v_mov_b32_e32 v32, 0
-; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_nop 11
+; GFX942-NEXT: s_nop 13
; GFX942-NEXT: global_store_dwordx4 v32, v[28:31], s[0:1] offset:112
; GFX942-NEXT: global_store_dwordx4 v32, v[24:27], s[0:1] offset:96
; GFX942-NEXT: global_store_dwordx4 v32, v[20:23], s[0:1] offset:80
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index 6ebb6aaa4520f..1d37813f62256 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -173,10 +173,10 @@ define amdgpu_kernel void @s_test_imin_sle_i32(ptr addrspace(1) %out, i32 %a, i3
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_min_i32 s2, s2, s3
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_min_i32 s0, s2, s3
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
@@ -187,10 +187,10 @@ define amdgpu_kernel void @s_test_imin_sle_i32(ptr addrspace(1) %out, i32 %a, i3
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_min_i32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_min_i32 s0, s2, s3
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -261,10 +261,10 @@ define amdgpu_kernel void @s_test_imin_sle_v1i32(ptr addrspace(1) %out, <1 x i32
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_min_i32 s2, s2, s3
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_min_i32 s0, s2, s3
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
@@ -275,10 +275,10 @@ define amdgpu_kernel void @s_test_imin_sle_v1i32(ptr addrspace(1) %out, <1 x i32
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_min_i32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_min_i32 s0, s2, s3
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -913,18 +913,18 @@ define amdgpu_kernel void @s_test_imin_sle_v2i16(ptr addrspace(1) %out, <2 x i16
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_ashr_i32 s4, s2, 16
-; CI-NEXT: s_sext_i32_i16 s2, s2
-; CI-NEXT: s_ashr_i32 s5, s3, 16
-; CI-NEXT: s_sext_i32_i16 s3, s3
-; CI-NEXT: s_min_i32 s4, s4, s5
-; CI-NEXT: s_min_i32 s2, s2, s3
-; CI-NEXT: s_lshl_b32 s3, s4, 16
-; CI-NEXT: s_and_b32 s2, s2, 0xffff
-; CI-NEXT: s_or_b32 s2, s2, s3
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: s_ashr_i32 s0, s2, 16
+; CI-NEXT: s_sext_i32_i16 s1, s2
+; CI-NEXT: s_ashr_i32 s2, s3, 16
+; CI-NEXT: s_sext_i32_i16 s3, s3
+; CI-NEXT: s_min_i32 s0, s0, s2
+; CI-NEXT: s_min_i32 s1, s1, s3
+; CI-NEXT: s_lshl_b32 s0, s0, 16
+; CI-NEXT: s_and_b32 s1, s1, 0xffff
+; CI-NEXT: s_or_b32 s0, s1, s0
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
@@ -935,18 +935,18 @@ define amdgpu_kernel void @s_test_imin_sle_v2i16(ptr addrspace(1) %out, <2 x i16
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ashr_i32 s4, s3, 16
-; VI-NEXT: s_ashr_i32 s5, s2, 16
-; VI-NEXT: s_sext_i32_i16 s3, s3
-; VI-NEXT: s_sext_i32_i16 s2, s2
-; VI-NEXT: s_min_i32 s4, s5, s4
-; VI-NEXT: s_min_i32 s2, s2, s3
-; VI-NEXT: s_lshl_b32 s3, s4, 16
-; VI-NEXT: s_and_b32 s2, s2, 0xffff
-; VI-NEXT: s_or_b32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_ashr_i32 s0, s3, 16
+; VI-NEXT: s_ashr_i32 s1, s2, 16
+; VI-NEXT: s_sext_i32_i16 s3, s3
+; VI-NEXT: s_min_i32 s0, s1, s0
+; VI-NEXT: s_sext_i32_i16 s1, s2
+; VI-NEXT: s_min_i32 s1, s1, s3
+; VI-NEXT: s_lshl_b32 s0, s0, 16
+; VI-NEXT: s_and_b32 s1, s1, 0xffff
+; VI-NEXT: s_or_b32 s0, s1, s0
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -1074,118 +1074,118 @@ define amdgpu_kernel void @s_test_imin_sle_v4i16(ptr addrspace(1) %out, <4 x i16
;
; CI-LABEL: s_test_imin_sle_v4i16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x4
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_ashr_i32 s6, s0, 16
-; CI-NEXT: s_ashr_i32 s7, s1, 16
-; CI-NEXT: s_sext_i32_i16 s0, s0
-; CI-NEXT: s_sext_i32_i16 s1, s1
-; CI-NEXT: s_ashr_i32 s8, s2, 16
-; CI-NEXT: s_ashr_i32 s9, s3, 16
-; CI-NEXT: s_sext_i32_i16 s2, s2
-; CI-NEXT: s_sext_i32_i16 s3, s3
-; CI-NEXT: s_min_i32 s7, s7, s9
-; CI-NEXT: s_min_i32 s1, s1, s3
-; CI-NEXT: s_min_i32 s3, s6, s8
-; CI-NEXT: s_min_i32 s0, s0, s2
-; CI-NEXT: s_lshl_b32 s7, s7, 16
-; CI-NEXT: s_and_b32 s1, s1, 0xffff
-; CI-NEXT: s_lshl_b32 s3, s3, 16
-; CI-NEXT: s_and_b32 s0, s0, 0xffff
-; CI-NEXT: s_or_b32 s1, s1, s7
-; CI-NEXT: s_or_b32 s0, s0, s3
-; CI-NEXT: v_mov_b32_e32 v2, s4
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v3, s5
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT: s_ashr_i32 s0, s2, 16
+; CI-NEXT: s_ashr_i32 s1, s3, 16
+; CI-NEXT: s_sext_i32_i16 s2, s2
+; CI-NEXT: s_sext_i32_i16 s3, s3
+; CI-NEXT: s_ashr_i32 s6, s4, 16
+; CI-NEXT: s_ashr_i32 s7, s5, 16
+; CI-NEXT: s_sext_i32_i16 s4, s4
+; CI-NEXT: s_sext_i32_i16 s5, s5
+; CI-NEXT: s_min_i32 s1, s1, s7
+; CI-NEXT: s_min_i32 s3, s3, s5
+; CI-NEXT: s_min_i32 s0, s0, s6
+; CI-NEXT: s_min_i32 s2, s2, s4
+; CI-NEXT: s_lshl_b32 s1, s1, 16
+; CI-NEXT: s_and_b32 s3, s3, 0xffff
+; CI-NEXT: s_lshl_b32 s0, s0, 16
+; CI-NEXT: s_and_b32 s2, s2, 0xffff
+; CI-NEXT: s_or_b32 s1, s3, s1
+; CI-NEXT: s_or_b32 s0, s2, s0
+; CI-NEXT: v_mov_b32_e32 v2, s0
+; CI-NEXT: v_mov_b32_e32 v3, s1
+; CI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_test_imin_sle_v4i16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ashr_i32 s6, s3, 16
-; VI-NEXT: s_ashr_i32 s7, s1, 16
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_ashr_i32 s0, s5, 16
+; VI-NEXT: s_ashr_i32 s1, s3, 16
+; VI-NEXT: s_min_i32 s0, s1, s0
+; VI-NEXT: s_sext_i32_i16 s1, s5
; VI-NEXT: s_sext_i32_i16 s3, s3
-; VI-NEXT: s_sext_i32_i16 s1, s1
-; VI-NEXT: s_min_i32 s6, s7, s6
-; VI-NEXT: s_min_i32 s1, s1, s3
-; VI-NEXT: s_lshl_b32 s6, s6, 16
+; VI-NEXT: s_min_i32 s1, s3, s1
+; VI-NEXT: s_lshl_b32 s0, s0, 16
; VI-NEXT: s_and_b32 s1, s1, 0xffff
-; VI-NEXT: s_or_b32 s1, s1, s6
+; VI-NEXT: s_or_b32 s0, s1, s0
+; VI-NEXT: s_ashr_i32 s1, s4, 16
; VI-NEXT: s_ashr_i32 s3, s2, 16
-; VI-NEXT: s_ashr_i32 s6, s0, 16
+; VI-NEXT: s_min_i32 s1, s3, s1
+; VI-NEXT: s_sext_i32_i16 s3, s4
; VI-NEXT: s_sext_i32_i16 s2, s2
-; VI-NEXT: s_sext_i32_i16 s0, s0
-; VI-NEXT: s_min_i32 s3, s6, s3
-; VI-NEXT: s_min_i32 s0, s0, s2
-; VI-NEXT: s_lshl_b32 s3, s3, 16
-; VI-NEXT: s_and_b32 s0, s0, 0xffff
-; VI-NEXT: s_or_b32 s0, s0, s3
-; VI-NEXT: v_mov_b32_e32 v2, s4
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_min_i32 s2, s2, s3
+; VI-NEXT: s_lshl_b32 s1, s1, 16
+; VI-NEXT: s_and_b32 s2, s2, 0xffff
+; VI-NEXT: s_or_b32 s1, s2, s1
+; VI-NEXT: v_mov_b32_e32 v2, s1
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_test_imin_sle_v4i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
-; GFX9-NEXT: v_pk_min_i16 v1, s1, v0
-; GFX9-NEXT: v_pk_min_i16 v0, s0, v3
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
+; GFX9-NEXT: v_mov_b32_e32 v0, s5
+; GFX9-NEXT: v_mov_b32_e32 v3, s4
+; GFX9-NEXT: v_pk_min_i16 v1, s3, v0
+; GFX9-NEXT: v_pk_min_i16 v0, s2, v3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: s_test_imin_sle_v4i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_pk_min_i16 v1, s1, s3
-; GFX10-NEXT: v_pk_min_i16 v0, s0, s2
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
+; GFX10-NEXT: v_pk_min_i16 v1, s3, s5
+; GFX10-NEXT: v_pk_min_i16 v0, s2, s4
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_imin_sle_v4i16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x10
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_pk_min_i16 v1, s1, s3
-; GFX11-NEXT: v_pk_min_i16 v0, s0, s2
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: v_pk_min_i16 v1, s3, s5
+; GFX11-NEXT: v_pk_min_i16 v0, s2, s4
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX1250-LABEL: s_test_imin_sle_v4i16:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_pk_min_i16 v1, s1, s3
-; GFX1250-NEXT: v_pk_min_i16 v0, s0, s2
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[6:7]
+; GFX1250-NEXT: v_pk_min_i16 v1, s3, s7
+; GFX1250-NEXT: v_pk_min_i16 v0, s2, s6
+; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
%cmp = icmp sle <4 x i16> %a, %b
%val = select <4 x i1> %cmp, <4 x i16> %a, <4 x i16> %b
@@ -1552,10 +1552,10 @@ define amdgpu_kernel void @s_test_imin_slt_i32(ptr addrspace(1) %out, i32 %a, i3
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_min_i32 s2, s2, s3
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_min_i32 s0, s2, s3
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
@@ -1566,10 +1566,10 @@ define amdgpu_kernel void @s_test_imin_slt_i32(ptr addrspace(1) %out, i32 %a, i3
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_min_i32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_min_i32 s0, s2, s3
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -1636,92 +1636,92 @@ define amdgpu_kernel void @s_test_imin_slt_v2i32(ptr addrspace(1) %out, <2 x i32
;
; CI-LABEL: s_test_imin_slt_v2i32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x4
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_min_i32 s1, s1, s3
-; CI-NEXT: s_min_i32 s0, s0, s2
-; CI-NEXT: v_mov_b32_e32 v2, s4
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v3, s5
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT: s_min_i32 s0, s3, s5
+; CI-NEXT: s_min_i32 s1, s2, s4
+; CI-NEXT: v_mov_b32_e32 v2, s1
+; CI-NEXT: v_mov_b32_e32 v3, s0
+; CI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_test_imin_slt_v2i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_min_i32 s1, s1, s3
-; VI-NEXT: s_min_i32 s0, s0, s2
-; VI-NEXT: v_mov_b32_e32 v2, s4
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v3, s5
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_min_i32 s0, s3, s5
+; VI-NEXT: s_min_i32 s1, s2, s4
+; VI-NEXT: v_mov_b32_e32 v2, s1
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_test_imin_slt_v2i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_min_i32 s1, s1, s3
-; GFX9-NEXT: s_min_i32 s0, s0, s2
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
+; GFX9-NEXT: s_min_i32 s3, s3, s5
+; GFX9-NEXT: s_min_i32 s2, s2, s4
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s3
+; GFX9-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: s_test_imin_slt_v2i32:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x10
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_min_i32 s0, s0, s2
-; GFX10-NEXT: s_min_i32 s1, s1, s3
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[4:5]
+; GFX10-NEXT: s_min_i32 s2, s2, s4
+; GFX10-NEXT: s_min_i32 s3, s3, s5
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_imin_slt_v2i32:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x10
; GFX11-NEXT: v_mov_b32_e32 v2, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_min_i32 s0, s0, s2
-; GFX11-NEXT: s_min_i32 s1, s1, s3
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: v_mov_b32_e32 v1, s1
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: s_min_i32 s2, s2, s4
+; GFX11-NEXT: s_min_i32 s3, s3, s5
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: v_mov_b32_e32 v1, s3
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
;
; GFX1250-LABEL: s_test_imin_slt_v2i32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x10 nv
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_min_i32 s0, s0, s2
-; GFX1250-NEXT: s_min_i32 s1, s1, s3
-; GFX1250-NEXT: v_mov_b32_e32 v0, s0
-; GFX1250-NEXT: v_mov_b32_e32 v1, s1
-; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[6:7]
+; GFX1250-NEXT: s_min_i32 s2, s2, s6
+; GFX1250-NEXT: s_min_i32 s3, s3, s7
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: v_mov_b32_e32 v1, s3
+; GFX1250-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX1250-NEXT: s_endpgm
%cmp = icmp slt <2 x i32> %a, %b
%val = select <2 x i1> %cmp, <2 x i32> %a, <2 x i32> %b
@@ -1743,38 +1743,35 @@ define amdgpu_kernel void @s_test_imin_slt_imm_i32(ptr addrspace(1) %out, i32 %a
;
; CI-LABEL: s_test_imin_slt_imm_i32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_min_i32 s2, s2, 8
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_min_i32 s0, s2, 8
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_test_imin_slt_imm_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_min_i32 s2, s2, 8
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_min_i32 s0, s2, 8
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_test_imin_slt_imm_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_min_i32 s2, s2, 8
@@ -1784,9 +1781,7 @@ define amdgpu_kernel void @s_test_imin_slt_imm_i32(ptr addrspace(1) %out, i32 %a
;
; GFX10-LABEL: s_test_imin_slt_imm_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_min_i32 s2, s2, 8
@@ -1796,9 +1791,7 @@ define amdgpu_kernel void @s_test_imin_slt_imm_i32(ptr addrspace(1) %out, i32 %a
;
; GFX11-LABEL: s_test_imin_slt_imm_i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_min_i32 s2, s2, 8
@@ -1838,38 +1831,35 @@ define amdgpu_kernel void @s_test_imin_sle_imm_i32(ptr addrspace(1) %out, i32 %a
;
; CI-LABEL: s_test_imin_sle_imm_i32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_min_i32 s2, s2, 8
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_min_i32 s0, s2, 8
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_test_imin_sle_imm_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_min_i32 s2, s2, 8
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_min_i32 s0, s2, 8
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_test_imin_sle_imm_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_min_i32 s2, s2, 8
@@ -1879,9 +1869,7 @@ define amdgpu_kernel void @s_test_imin_sle_imm_i32(ptr addrspace(1) %out, i32 %a
;
; GFX10-LABEL: s_test_imin_sle_imm_i32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_min_i32 s2, s2, 8
@@ -1891,9 +1879,7 @@ define amdgpu_kernel void @s_test_imin_sle_imm_i32(ptr addrspace(1) %out, i32 %a
;
; GFX11-LABEL: s_test_imin_sle_imm_i32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_min_i32 s2, s2, 8
@@ -2452,10 +2438,10 @@ define amdgpu_kernel void @s_test_umin_ule_i32(ptr addrspace(1) %out, i32 %a, i3
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_min_u32 s2, s2, s3
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_min_u32 s0, s2, s3
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
@@ -2466,10 +2452,10 @@ define amdgpu_kernel void @s_test_umin_ule_i32(ptr addrspace(1) %out, i32 %a, i3
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_min_u32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_min_u32 s0, s2, s3
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -2863,10 +2849,10 @@ define amdgpu_kernel void @s_test_umin_ult_i32(ptr addrspace(1) %out, i32 %a, i3
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_min_u32 s2, s2, s3
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_min_u32 s0, s2, s3
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
@@ -2877,10 +2863,10 @@ define amdgpu_kernel void @s_test_umin_ult_i32(ptr addrspace(1) %out, i32 %a, i3
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_min_u32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_min_u32 s0, s2, s3
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -3316,10 +3302,10 @@ define amdgpu_kernel void @s_test_umin_ult_v1i32(ptr addrspace(1) %out, <1 x i32
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_min_u32 s2, s2, s3
; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: s_min_u32 s0, s2, s3
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_dword v[0:1], v2
; CI-NEXT: s_endpgm
;
@@ -3330,10 +3316,10 @@ define amdgpu_kernel void @s_test_umin_ult_v1i32(ptr addrspace(1) %out, <1 x i32
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_min_u32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_min_u32 s0, s2, s3
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -4187,42 +4173,39 @@ define amdgpu_kernel void @s_test_imin_sle_i16(ptr addrspace(1) %out, i16 %a, i1
;
; CI-LABEL: s_test_imin_sle_i16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CI-NEXT: s_add_i32 s12, s12, s17
; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_sext_i32_i16 s3, s2
-; CI-NEXT: s_ashr_i32 s2, s2, 16
-; CI-NEXT: s_min_i32 s2, s3, s2
; CI-NEXT: v_mov_b32_e32 v0, s0
; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: s_sext_i32_i16 s0, s2
+; CI-NEXT: s_ashr_i32 s1, s2, 16
+; CI-NEXT: s_min_i32 s0, s0, s1
+; CI-NEXT: v_mov_b32_e32 v2, s0
; CI-NEXT: flat_store_short v[0:1], v2
; CI-NEXT: s_endpgm
;
; VI-LABEL: s_test_imin_sle_i16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; VI-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; VI-NEXT: s_add_i32 s12, s12, s17
; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ashr_i32 s3, s2, 16
-; VI-NEXT: s_sext_i32_i16 s2, s2
-; VI-NEXT: s_min_i32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_ashr_i32 s0, s2, 16
+; VI-NEXT: s_sext_i32_i16 s1, s2
+; VI-NEXT: s_min_i32 s0, s1, s0
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_test_imin_sle_i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_ashr_i32 s3, s2, 16
@@ -4234,9 +4217,7 @@ define amdgpu_kernel void @s_test_imin_sle_i16(ptr addrspace(1) %out, i16 %a, i1
;
; GFX10-LABEL: s_test_imin_sle_i16:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[8:9], 0x8
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GFX10-NEXT: v_mov_b32_e32 v0, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_ashr_i32 s3, s2, 16
@@ -4248,9 +4229,7 @@ define amdgpu_kernel void @s_test_imin_sle_i16(ptr addrspace(1) %out, i16 %a, i1
;
; GFX11-LABEL: s_test_imin_sle_i16:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v0, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_ashr_i32 s3, s2, 16
diff --git a/llvm/test/CodeGen/AMDGPU/misched-remat-revert.ll b/llvm/test/CodeGen/AMDGPU/misched-remat-revert.ll
new file mode 100644
index 0000000000000..9237ebb0df566
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/misched-remat-revert.ll
@@ -0,0 +1,576 @@
+; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -O3 -stop-after=machine-scheduler < %s | FileCheck %s
+
+; This testcase hit a situation where reverting scheduling after the scheduler's
+; rematerialization stage would cause incoherent MI and slot orders, hitting an
+; assert later on in the RP trackers. This is fixed by ensuring that all MIs that
+; should be alive at the end of the stage are marked non-debug before scheduling
+; is reverted, but is extremely sensitive to scheduling and rematerialization
+; decisions.
+
+%f8 = type { i8 }
+
+ at shared = external addrspace(3) global [16384 x i8]
+
+define amdgpu_kernel void @test_revert_schedule(i32 %arg0, i32 %arg1, ptr addrspace(3) %p15, ptr addrspace(3) %lds, ptr addrspace(3) %arg, ptr addrspace(3) %p14, i32 %arg2, ptr addrspace(3) %arg3, ptr addrspace(3) %arg4, i32 %arg5, i32 %arg6, ptr addrspace(3) %p12, i32 %x7, ptr addrspace(3) %p7, i32 %a7, ptr addrspace(3) %arg7, i1 %loopcond, i32 %a5, i32 %a3, i32 %a4, i32 %a2, <4 x i8> %arg8, <4 x i8> %arg9) #0 {
+ ; CHECK-LABEL: name: test_revert_schedule
+ ; CHECK: bb.0.entry:
+ ; CHECK-NEXT: successors: %bb.1(0x80000000)
+ ; CHECK-NEXT: liveins: $vgpr0, $sgpr4_sgpr5
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_32(s32) = COPY $vgpr0
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 64, 0 :: (dereferenceable invariant load (s32) from %ir.loopcond.kernarg.offset.align.down, align 16, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 0, 0 :: (dereferenceable invariant load (s128) from %ir.arg0.kernarg.offset1, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM1:%[0-9]+]]:sgpr_32 = S_LOAD_DWORD_IMM [[COPY]](p4), 16, 0 :: (dereferenceable invariant load (s32) from %ir.arg0.kernarg.offset1 + 16, align 16, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 32, 0 :: (dereferenceable invariant load (s32) from %ir.arg4.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: S_BITCMP1_B32 [[S_LOAD_DWORD_IMM]], 0, implicit-def $scc
+ ; CHECK-NEXT: [[S_CSELECT_B64_:%[0-9]+]]:sreg_64_xexec = S_CSELECT_B64 -1, 0, implicit $scc
+ ; CHECK-NEXT: [[V_AND_B32_e32_:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 1, [[COPY1]](s32), implicit $exec
+ ; CHECK-NEXT: [[V_LSHLREV_B32_e32_:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e32 1, [[V_AND_B32_e32_]], implicit $exec
+ ; CHECK-NEXT: [[V_AND_B32_e32_1:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 3, [[COPY1]](s32), implicit $exec
+ ; CHECK-NEXT: [[V_OR_B32_e32_:%[0-9]+]]:vgpr_32 = V_OR_B32_e32 20, [[V_AND_B32_e32_1]], implicit $exec
+ ; CHECK-NEXT: [[V_OR_B32_e32_1:%[0-9]+]]:vgpr_32 = V_OR_B32_e32 21, [[V_AND_B32_e32_1]], implicit $exec
+ ; CHECK-NEXT: [[V_XOR_B32_e32_:%[0-9]+]]:vgpr_32 = V_XOR_B32_e32 1, [[COPY1]](s32), implicit $exec
+ ; CHECK-NEXT: [[V_XOR_B32_e32_1:%[0-9]+]]:vgpr_32 = V_XOR_B32_e32 1, [[V_AND_B32_e32_1]], implicit $exec
+ ; CHECK-NEXT: [[V_AND_B32_e32_2:%[0-9]+]]:vgpr_32 = V_AND_B32_e32 31, [[COPY1]](s32), implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_SUB_U32_e32_:%[0-9]+]]:vgpr_32 = V_SUB_U32_e32 0, [[V_XOR_B32_e32_1]], implicit $exec
+ ; CHECK-NEXT: [[V_LSHLREV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e32 1, [[V_SUB_U32_e32_]], implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec, implicit $exec
+ ; CHECK-NEXT: undef [[S_MOV_B32_:%[0-9]+]].sub0:sgpr_128 = S_MOV_B32 0
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub0
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_]].sub0
+ ; CHECK-NEXT: [[S_MOV_B32_:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_]].sub0
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:av_128_align2 = COPY [[S_MOV_B32_]]
+ ; CHECK-NEXT: [[V_MOV_B:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 0, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 12288, implicit $exec
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:vgpr_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
+ ; CHECK-NEXT: [[V_ADD_U32_e32_:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORD_IMM2]], [[V_XOR_B32_e32_]], implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e32_1:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORD_IMM2]], [[V_LSHLREV_B32_e32_1]], implicit $exec
+ ; CHECK-NEXT: [[S_XOR_B64_:%[0-9]+]]:sreg_64_xexec = S_XOR_B64 [[S_CSELECT_B64_]], -1, implicit-def dead $scc
+ ; CHECK-NEXT: [[V_CNDMASK_B32_e64_:%[0-9]+]]:vgpr_32 = V_CNDMASK_B32_e64 0, 0, 0, 1, [[S_XOR_B64_]], implicit $exec
+ ; CHECK-NEXT: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64_xexec = V_CMP_NE_U32_e64 1, [[V_CNDMASK_B32_e64_]], implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_3:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_4:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1.loop:
+ ; CHECK-NEXT: successors: %bb.2(0x04000000), %bb.1(0x7c000000)
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[V_ADD_U32_e32_2:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORDX4_IMM]].sub3, [[V_MOV_B32_e32_1]], implicit $exec
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET [[S_MOV_B32_]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET1:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET [[S_MOV_B32_]], 0, 1, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[V_MOV_B32_e32_2]], [[S_MOV_B32_]], 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN1:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[V_MOV_B32_e32_2]], [[S_MOV_B32_]], 0, 3072, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 [[V_ADD_U32_e32_2]], [[COPY2]], 0, 0, implicit $exec :: (store (s128) into %ir.p19, addrspace 3)
+ ; CHECK-NEXT: S_WAITCNT .Vmcnt_0_Expcnt_0_Lgkmcnt_0
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET2:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET [[S_MOV_B32_]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET3:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET [[S_MOV_B32_]], 0, 1, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN2:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY3]], [[S_MOV_B32_]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub0:vreg_64_align2 = COPY [[S_MOV_B32_]].sub0
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B32_e32_4]], [[V_MOV_B]], 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub0:vreg_64_align2 = COPY [[S_MOV_B32_]].sub0
+ ; CHECK-NEXT: undef [[DS_READ_B32_gfx9_:%[0-9]+]].sub0:vreg_64_align2 = DS_READ_B32_gfx9 [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) null`, align 8, addrspace 3)
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_4:%[0-9]+]].sub1:vreg_64_align2 = DS_READ_B32_gfx9 [[V_ADD_U32_e32_]], 0, 0, implicit $exec :: (load (s32) from %ir.sunkaddr, addrspace 3)
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_1:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B32_e32_3]], [[V_MOV_B]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: undef [[V_MOV_B32_e32_3:%[0-9]+]].sub1:vreg_64_align2 = DS_READ_B32_gfx9 [[V_ADD_U32_e32_1]], 0, 0, implicit $exec :: (load (s32) from %ir.sunkaddr2, addrspace 3)
+ ; CHECK-NEXT: SCHED_GROUP_BARRIER 0, 0, 0
+ ; CHECK-NEXT: $vcc = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc
+ ; CHECK-NEXT: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = COPY [[V_AND_B32_e32_2]]
+ ; CHECK-NEXT: S_CBRANCH_VCCNZ %bb.1, implicit $vcc
+ ; CHECK-NEXT: S_BRANCH %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2.exit:
+ ; CHECK-NEXT: [[V_MOV_B1:%[0-9]+]]:vreg_64_align2 = V_MOV_B64_PSEUDO 0, implicit $exec
+ ; CHECK-NEXT: undef [[S_MOV_B32_1:%[0-9]+]].sub0:sgpr_128 = S_MOV_B32 0
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_2:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_4:%[0-9]+]].sub0:vreg_64_align2 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: undef [[S_MOV_B32_2:%[0-9]+]].sub0:sreg_64 = S_MOV_B32 16843009
+ ; CHECK-NEXT: [[S_MOV_B32_2:%[0-9]+]].sub1:sreg_64 = COPY [[S_MOV_B32_2]].sub0
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_3:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B32_e32_4]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_2]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:av_64_align2 = COPY [[S_MOV_B32_2]]
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_:%[0-9]+]].sub1:vreg_64_align2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_4:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[COPY4]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_3]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 24, 0 :: (dereferenceable invariant load (s32) from %ir.arg2.kernarg.offset, align 8, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 40, 0 :: (dereferenceable invariant load (s32) from %ir.arg6.kernarg.offset, align 8, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM5:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 48, 0 :: (dereferenceable invariant load (s32) from %ir.x7.kernarg.offset, align 16, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM6:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 56, 0 :: (dereferenceable invariant load (s32) from %ir.a7.kernarg.offset, align 8, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM7:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[COPY]](p4), 72, 0 :: (dereferenceable invariant load (s32) from %ir.a3.kernarg.offset, align 8, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM1:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY]](p4), 80, 0 :: (dereferenceable invariant load (s128) from %ir.a2.kernarg.offset, addrspace 4)
+ ; CHECK-NEXT: KILL [[COPY]](p4)
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_5:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_4]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_5:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 16384, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_6:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 20480, implicit $exec
+ ; CHECK-NEXT: undef [[COPY5:%[0-9]+]].sub1:vreg_64_align2 = COPY [[DS_READ_B32_gfx9_]].sub1
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_6:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[DS_READ_B32_gfx9_]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_5]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: undef [[COPY6:%[0-9]+]].sub0:vreg_64_align2 = COPY [[DS_READ_B32_gfx9_]].sub1
+ ; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]].sub2:sgpr_128 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_7:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_6]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[S_MOV_B32_1:%[0-9]+]].sub3:sgpr_128 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN3:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[V_MOV_B32_e32_5]], [[S_MOV_B32_1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN4:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[V_MOV_B32_e32_5]], [[S_MOV_B32_1]], 0, 1024, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_8:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_7]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]].sub0:vreg_64_align2 = COPY [[S_LOAD_DWORDX4_IMM1]].sub1
+ ; CHECK-NEXT: [[COPY6:%[0-9]+]].sub1:vreg_64_align2 = COPY [[S_LOAD_DWORDX4_IMM1]].sub2
+ ; CHECK-NEXT: [[COPY7:%[0-9]+]]:vgpr_32 = COPY [[S_LOAD_DWORD_IMM5]]
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_9:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFSET]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_8]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY8:%[0-9]+]]:vgpr_32 = COPY [[S_LOAD_DWORD_IMM3]]
+ ; CHECK-NEXT: [[S_ASHR_I32_:%[0-9]+]]:sreg_32 = S_ASHR_I32 [[S_LOAD_DWORD_IMM4]], 31, implicit-def dead $scc
+ ; CHECK-NEXT: [[COPY9:%[0-9]+]]:vgpr_32 = COPY [[S_LOAD_DWORD_IMM6]]
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_10:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFSET]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_9]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN5:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY7]], [[S_MOV_B32_1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN6:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[V_MOV_B32_e32_5]], [[S_MOV_B32_1]], 0, 3072, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: KILL [[COPY7]]
+ ; CHECK-NEXT: KILL [[V_MOV_B32_e32_5]]
+ ; CHECK-NEXT: [[S_LSHR_B32_:%[0-9]+]]:sreg_32 = S_LSHR_B32 [[S_ASHR_I32_]], 26, implicit-def dead $scc
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_11:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFSET1]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_10]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN7:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY9]], [[S_MOV_B32_1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN8:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[V_MOV_B32_e32_6]], [[S_MOV_B32_1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: KILL [[COPY9]]
+ ; CHECK-NEXT: [[S_ADD_I32_:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_LOAD_DWORD_IMM4]], [[S_LSHR_B32_]], implicit-def dead $scc
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_12:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFSET1]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_11]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[S_ASHR_I32_1:%[0-9]+]]:sreg_32 = S_ASHR_I32 [[S_ADD_I32_]], 6, implicit-def dead $scc
+ ; CHECK-NEXT: [[S_LSHL_B32_:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[S_ASHR_I32_1]], 1, implicit-def dead $scc
+ ; CHECK-NEXT: [[S_OR_B32_:%[0-9]+]]:sreg_32 = S_OR_B32 [[S_LSHL_B32_]], 19456, implicit-def dead $scc
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_13:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_12]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY10:%[0-9]+]]:vgpr_32 = COPY [[S_OR_B32_]]
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN9:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY8]], [[S_MOV_B32_1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN10:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY10]], [[S_MOV_B32_1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: KILL [[COPY10]]
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_14:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_13]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET4:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET [[S_MOV_B32_1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFSET5:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFSET [[S_MOV_B32_1]], 0, 1, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[COPY11:%[0-9]+]]:vgpr_32 = COPY [[S_LOAD_DWORD_IMM7]]
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_15:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_1]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[COPY12:%[0-9]+]]:vgpr_32 = COPY [[S_LOAD_DWORDX4_IMM1]].sub0
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN11:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY11]], [[S_MOV_B32_1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN12:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY12]], [[S_MOV_B32_1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: KILL [[COPY11]]
+ ; CHECK-NEXT: KILL [[COPY12]]
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_16:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[COPY5]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_15]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN13:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[V_MOV_B32_e32_6]], [[S_MOV_B32_1]], 0, 2048, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN14:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[V_MOV_B32_e32_6]], [[S_MOV_B32_1]], 0, 3072, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: KILL [[V_MOV_B32_e32_6]]
+ ; CHECK-NEXT: [[COPY13:%[0-9]+]]:vgpr_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub0
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_17:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[COPY6]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_16]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[BUFFER_LOAD_DWORDX4_OFFEN15:%[0-9]+]]:vreg_128_align2 = BUFFER_LOAD_DWORDX4_OFFEN [[COPY13]], [[S_MOV_B32_1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s128) from `ptr addrspace(8) null`, align 1, addrspace 8)
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_18:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_17]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MOV_B32_e32_3:%[0-9]+]].sub0:vreg_64_align2 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: [[V_OR_B32_e32_2:%[0-9]+]]:vgpr_32 = V_OR_B32_e32 [[S_LOAD_DWORDX4_IMM]].sub0, [[COPY1]](s32), implicit $exec
+ ; CHECK-NEXT: [[V_OR_B32_e32_3:%[0-9]+]]:vgpr_32 = V_OR_B32_e32 1, [[COPY1]](s32), implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_19:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_18]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_OR_B32_e32_4:%[0-9]+]]:vgpr_32 = V_OR_B32_e32 24, [[V_AND_B32_e32_1]], implicit $exec
+ ; CHECK-NEXT: [[V_XOR_B32_e32_2:%[0-9]+]]:vgpr_32 = V_XOR_B32_e32 [[V_XOR_B32_e32_]], [[V_OR_B32_e32_3]], implicit $exec
+ ; CHECK-NEXT: [[V_OR_B32_e32_5:%[0-9]+]]:vgpr_32 = V_OR_B32_e32 [[V_XOR_B32_e32_2]], [[V_XOR_B32_e32_1]], implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_20:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_19]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_LSHLREV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e32 1, [[V_OR_B32_e32_5]], implicit $exec
+ ; CHECK-NEXT: [[V_XOR_B32_e32_3:%[0-9]+]]:vgpr_32 = V_XOR_B32_e32 1, [[V_OR_B32_e32_]], implicit $exec
+ ; CHECK-NEXT: [[V_OR_B32_e32_6:%[0-9]+]]:vgpr_32 = V_OR_B32_e32 [[S_LOAD_DWORDX4_IMM]].sub0, [[V_XOR_B32_e32_3]], implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_21:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B32_e32_3]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_20]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_XOR_B32_e32_4:%[0-9]+]]:vgpr_32 = V_XOR_B32_e32 1, [[V_OR_B32_e32_4]], implicit $exec
+ ; CHECK-NEXT: [[V_LSHRREV_B32_e32_:%[0-9]+]]:vgpr_32 = V_LSHRREV_B32_e32 3, [[COPY1]](s32), implicit $exec
+ ; CHECK-NEXT: [[V_LSHLREV_B32_e32_3:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e32 1, [[COPY1]](s32), implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_22:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_21]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 [[DS_READ_B32_gfx9_]].sub1, [[BUFFER_LOAD_DWORDX4_OFFSET2]], 0, 0, implicit $exec :: (store (s128) into `ptr addrspace(3) null`, addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 [[COPY8]], [[BUFFER_LOAD_DWORDX4_OFFEN2]], 0, 0, implicit $exec :: (store (s128) into %ir.p18, addrspace 3)
+ ; CHECK-NEXT: [[V_OR_B32_e32_7:%[0-9]+]]:vgpr_32 = V_OR_B32_e32 1, [[V_AND_B32_e32_2]], implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_23:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_22]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e32_3:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORD_IMM3]], [[V_OR_B32_e32_7]], implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B128_gfx9 [[V_ADD_U32_e32_3]], [[BUFFER_LOAD_DWORDX4_OFFSET3]], 512, 0, implicit $exec :: (store (s128) into %ir.14, addrspace 3)
+ ; CHECK-NEXT: [[COPY14:%[0-9]+]]:vgpr_32 = COPY [[S_LOAD_DWORDX4_IMM]].sub2
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_24:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_23]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_1:%[0-9]+]]:av_32 = DS_READ_B32_gfx9 [[COPY14]], 0, 0, implicit $exec :: (load (s32) from %ir.6, align 8, addrspace 3)
+ ; CHECK-NEXT: [[V_ADD_U32_e32_4:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORDX4_IMM]].sub1, [[V_LSHLREV_B32_e32_1]], implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e32_5:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORDX4_IMM]].sub1, [[V_OR_B32_e32_2]], implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_25:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_24]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: undef [[DS_READ_B32_gfx9_2:%[0-9]+]].sub1:av_64_align2 = DS_READ_B32_gfx9 [[V_ADD_U32_e32_4]], 0, 0, implicit $exec :: (load (s32) from %ir.p10, addrspace 3)
+ ; CHECK-NEXT: [[DS_READ_B64_gfx9_:%[0-9]+]]:av_64_align2 = DS_READ_B64_gfx9 [[V_ADD_U32_e32_5]], 0, 0, implicit $exec :: (load (s64) from %ir.p11, addrspace 3)
+ ; CHECK-NEXT: undef [[DS_READ_B32_gfx9_3:%[0-9]+]].sub1:av_64_align2 = DS_READ_B32_gfx9 [[DS_READ_B32_gfx9_]].sub1, 0, 0, implicit $exec :: (load (s32) from `ptr addrspace(3) null`, align 8, addrspace 3)
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_26:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_25]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_2:%[0-9]+]].sub0:av_64_align2 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: undef [[COPY15:%[0-9]+]].sub0:av_64_align2 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:av_64_align2 = COPY [[DS_READ_B64_gfx9_]].sub1
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_27:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_26]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e32_6:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORD_IMM1]], [[V_LSHLREV_B32_e32_]], implicit $exec
+ ; CHECK-NEXT: undef [[DS_READ_B32_gfx9_4:%[0-9]+]].sub1:av_64_align2 = DS_READ_B32_gfx9 [[V_ADD_U32_e32_6]], 0, 0, implicit $exec :: (load (s32) from %ir.p4, addrspace 3)
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_4:%[0-9]+]].sub0:av_64_align2 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_28:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_27]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e32_7:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORDX4_IMM]].sub3, [[V_SUB_U32_e32_]], implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e32_8:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORDX4_IMM]].sub1, [[V_ADD_U32_e32_7]], implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e32_9:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORDX4_IMM]].sub3, [[V_OR_B32_e32_1]], implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_29:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN1]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_28]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_ADD_U32_e32_10:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORDX4_IMM]].sub3, [[V_LSHLREV_B32_e32_]], implicit $exec
+ ; CHECK-NEXT: undef [[DS_READ_B32_gfx9_5:%[0-9]+]].sub1:av_64_align2 = DS_READ_B32_gfx9 [[V_ADD_U32_e32_8]], 0, 0, implicit $exec :: (load (s32) from %ir.p8, addrspace 3)
+ ; CHECK-NEXT: undef [[DS_READ_B32_gfx9_6:%[0-9]+]].sub1:av_64_align2 = DS_READ_B32_gfx9 [[V_ADD_U32_e32_9]], 0, 0, implicit $exec :: (load (s32) from %ir.sunkaddr3, addrspace 3)
+ ; CHECK-NEXT: undef [[DS_READ_B32_gfx9_7:%[0-9]+]].sub1:av_64_align2 = DS_READ_B32_gfx9 [[V_ADD_U32_e32_10]], 0, 0, implicit $exec :: (load (s32) from %ir.p5, addrspace 3)
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_3:%[0-9]+]].sub0:av_64_align2 = DS_READ_B32_gfx9 [[V_ADD_U32_e32_7]], 0, 0, implicit $exec :: (load (s32) from %ir.i85, align 8, addrspace 3)
+ ; CHECK-NEXT: [[V_ADD_U32_e32_11:%[0-9]+]]:vgpr_32 = V_ADD_U32_e32 [[S_LOAD_DWORD_IMM2]], [[V_OR_B32_e32_6]], implicit $exec
+ ; CHECK-NEXT: undef [[DS_READ_B32_gfx9_8:%[0-9]+]].sub0:av_64_align2 = DS_READ_B32_gfx9 [[V_ADD_U32_e32_11]], 0, 0, implicit $exec :: (load (s32) from %ir.p2, align 16, addrspace 3)
+ ; CHECK-NEXT: undef [[DS_READ_B32_gfx9_9:%[0-9]+]].sub1:av_64_align2 = DS_READ_B32_gfx9 [[V_LSHLREV_B32_e32_2]], 8192, 0, implicit $exec :: (load (s32) from %ir.p13, addrspace 3)
+ ; CHECK-NEXT: undef [[DS_READ_B32_gfx9_10:%[0-9]+]].sub0:av_64_align2 = DS_READ_B32_gfx9 [[V_XOR_B32_e32_4]], 8192, 0, implicit $exec :: (load (s32) from %ir.p9, align 16, addrspace 3)
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_5:%[0-9]+]].sub0:av_64_align2 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_6:%[0-9]+]].sub0:av_64_align2 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_7:%[0-9]+]].sub0:av_64_align2 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_9:%[0-9]+]].sub0:av_64_align2 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: SCHED_BARRIER 0
+ ; CHECK-NEXT: [[DS_READ_B64_gfx9_:%[0-9]+]].sub1:av_64_align2 = COPY [[DS_READ_B32_gfx9_1]]
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_30:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[COPY4]], [[BUFFER_LOAD_DWORDX4_OFFSET4]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_14]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_8:%[0-9]+]].sub1:av_64_align2 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: [[DS_READ_B32_gfx9_10:%[0-9]+]].sub1:av_64_align2 = COPY [[S_MOV_B32_1]].sub0
+ ; CHECK-NEXT: [[V_MUL_LO_U32_e64_:%[0-9]+]]:vgpr_32 = V_MUL_LO_U32_e64 [[V_LSHRREV_B32_e32_]], [[S_LOAD_DWORDX4_IMM]].sub0, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_31:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[DS_READ_B32_gfx9_2]], [[BUFFER_LOAD_DWORDX4_OFFEN4]].sub0_sub1, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: undef [[V_AND_OR_B32_e64_:%[0-9]+]].sub0:vreg_64_align2 = V_AND_OR_B32_e64 [[V_LSHLREV_B32_e32_3]], 56, [[V_MUL_LO_U32_e64_]], implicit $exec
+ ; CHECK-NEXT: [[V_AND_OR_B32_e64_1:%[0-9]+]]:vgpr_32 = V_AND_OR_B32_e64 [[COPY1]](s32), 48, [[V_AND_B32_e32_]], implicit $exec
+ ; CHECK-NEXT: [[V_LSHLREV_B32_e32_4:%[0-9]+]]:vgpr_32 = V_LSHLREV_B32_e32 2, [[V_AND_OR_B32_e64_1]], implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_32:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFSET4]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_30]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_33:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[COPY15]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_31]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_34:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN5]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_32]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_35:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[DS_READ_B64_gfx9_]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_33]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_36:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN5]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_34]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_37:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN6]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_35]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_38:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN9]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_36]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_39:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[DS_READ_B32_gfx9_4]], [[BUFFER_LOAD_DWORDX4_OFFEN6]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_37]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_40:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN9]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_38]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_41:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[DS_READ_B32_gfx9_5]], [[BUFFER_LOAD_DWORDX4_OFFEN8]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_39]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_42:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN10]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_40]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_43:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[DS_READ_B32_gfx9_3]], [[BUFFER_LOAD_DWORDX4_OFFEN8]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_41]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_44:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN10]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_42]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_45:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[DS_READ_B32_gfx9_6]], [[BUFFER_LOAD_DWORDX4_OFFSET5]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_43]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_46:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN7]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_44]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_47:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFSET5]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_45]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_48:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[DS_READ_B32_gfx9_9]], [[BUFFER_LOAD_DWORDX4_OFFEN7]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_46]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_49:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[DS_READ_B32_gfx9_7]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_47]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_50:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN11]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_48]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_51:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN13]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_49]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_52:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN11]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_50]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_53:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN3]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_29]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_54:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN14]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_51]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_55:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[DS_READ_B32_gfx9_10]], [[BUFFER_LOAD_DWORDX4_OFFEN12]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_52]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_56:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[DS_READ_B32_gfx9_8]], [[V_MOV_B1]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_53]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_57:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN15]].sub2_sub3, 0, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DS_READ_B32_gfx9_]].sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_57]].sub0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(3) null`, addrspace 3)
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_58:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN14]].sub2_sub3, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_54]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_59:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN15]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_55]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[V_LSHLREV_B32_e32_4]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_59]].sub0, 0, 0, implicit $exec :: (store (s32) into %ir.p21, addrspace 3)
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[DS_READ_B32_gfx9_]].sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_58]].sub0, 0, 0, implicit $exec :: (store (s32) into `ptr addrspace(3) null`, addrspace 3)
+ ; CHECK-NEXT: [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_60:%[0-9]+]]:vreg_128_align2 = V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64 [[V_MOV_B1]], [[BUFFER_LOAD_DWORDX4_OFFEN13]].sub0_sub1, [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_56]], 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: DS_WRITE_B32_gfx9 [[COPY14]], [[V_MFMA_F32_16X16X32_FP8_FP8_vgprcd_e64_60]].sub0, 0, 0, implicit $exec :: (store (s32) into %ir.6, addrspace 3)
+ ; CHECK-NEXT: [[V_AND_OR_B32_e64_:%[0-9]+]].sub1:vreg_64_align2 = V_ASHRREV_I32_e32 31, [[V_AND_OR_B32_e64_]].sub0, implicit $exec
+ ; CHECK-NEXT: [[V_LSHLREV_B64_e64_:%[0-9]+]]:vreg_64_align2 = V_LSHLREV_B64_e64 1, [[V_AND_OR_B32_e64_]], implicit $exec
+ ; CHECK-NEXT: GLOBAL_ATOMIC_PK_ADD_BF16 [[V_LSHLREV_B64_e64_]], [[DS_READ_B32_gfx9_]].sub1, 0, 0, implicit $exec :: (load store syncscope("agent") monotonic (s32) on %ir.p20, addrspace 1)
+ ; CHECK-NEXT: S_ENDPGM 0
+entry:
+ %i = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %i10 = lshr i32 %i, 3
+ %n6 = and i32 %i, 1
+ %m3 = shl i32 %n6, 1
+ %n4 = or i32 %i, %arg0
+ %d2 = or i32 %i, 1
+ %s = and i32 %i, 3
+ %d = or i32 %s, 20
+ %n = or i32 %d, 1
+ %i11 = getelementptr i8, ptr addrspace(3) %lds, i32 %n
+ %p6 = getelementptr %f8, ptr addrspace(3) %i11, i32 %m3
+ %i12 = getelementptr i8, ptr addrspace(3) %p6, i32 4
+ %d3 = or i32 %s, 24
+ %x = xor i32 %i, 1
+ %x2 = xor i32 %x, %d2
+ %x6 = xor i32 1, %s
+ %n2 = or i32 %x2, %x6
+ %i13 = shl i32 %n2, 1
+ %x4 = xor i32 1, %d
+ %n3 = or i32 %x4, %arg0
+ %x3 = xor i32 1, %d3
+ %x5 = and i32 %i, 31
+ br label %loop
+
+loop: ; preds = %loop, %entry
+ %phi0 = phi i32 [ 0, %entry ], [ %x5, %loop ]
+ %i14 = phi <4 x i8> [ zeroinitializer, %entry ], [ %i29, %loop ]
+ %i15 = phi <4 x i8> [ zeroinitializer, %entry ], [ %i27, %loop ]
+ %p19 = getelementptr %f8, ptr addrspace(3) %lds, i32 %phi0
+ store <4 x i32> zeroinitializer, ptr addrspace(3) %p19, align 16
+ %i16 = shufflevector <4 x i8> zeroinitializer, <4 x i8> %i15, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %ac = bitcast <16 x i8> %i16 to <2 x i64>
+ %i17 = shufflevector <4 x i8> zeroinitializer, <4 x i8> %i14, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %ac2 = bitcast <16 x i8> %i17 to <2 x i64>
+ %ae = extractelement <2 x i64> %ac, i64 0
+ %ae2 = extractelement <2 x i64> %ac2, i64 0
+ %i18 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae, i64 0, <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
+ %i19 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae2, i64 0, <4 x float> %i18, i32 0, i32 0, i32 0)
+ %i20 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 0, i32 0, i32 0)
+ %i21 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 1, i32 0, i32 0)
+ %i22 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 14336, i32 0, i32 0)
+ %i23 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 15360, i32 0, i32 0)
+ tail call void @llvm.amdgcn.s.waitcnt(i32 0)
+ %i24 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 0, i32 0, i32 0)
+ %i25 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 %arg0, i32 0, i32 0)
+ %i26 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 1, i32 0, i32 0)
+ %p3 = getelementptr i8, ptr addrspace(3) %arg4, i32 %x
+ %i27 = load <4 x i8>, ptr addrspace(3) %p3, align 4
+ %n5 = sub i32 0, %x6
+ %i28 = shl i32 %n5, 1
+ %p = getelementptr i8, ptr addrspace(3) %arg4, i32 %i28
+ %i29 = load <4 x i8>, ptr addrspace(3) %p, align 4
+ %i30 = load <4 x i8>, ptr addrspace(3) zeroinitializer, align 8
+ tail call void @llvm.amdgcn.sched.group.barrier(i32 0, i32 0, i32 0)
+ br i1 %loopcond, label %loop, label %exit
+
+exit: ; preds = %loop
+ %i31 = shl i32 %i, 1
+ %i32 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 0, i32 0, i32 0)
+ %i33 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 16384, i32 0, i32 0)
+ %i34 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 17408, i32 0, i32 0)
+ %i35 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 %x7, i32 0, i32 0)
+ %i36 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 %arg2, i32 0, i32 0)
+ %i37 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 19456, i32 0, i32 0)
+ %d4 = sdiv i32 %arg6, 64
+ %m5 = shl i32 %d4, 1
+ %a = or i32 19456, %m5
+ %i38 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 %a, i32 0, i32 0)
+ %i39 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 %a7, i32 0, i32 0)
+ %i40 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 20480, i32 0, i32 0)
+ %i41 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 1, i32 0, i32 0)
+ %i42 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 %a3, i32 0, i32 0)
+ %i43 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 %a2, i32 0, i32 0)
+ %i44 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 22528, i32 0, i32 0)
+ %i45 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 23552, i32 0, i32 0)
+ %i46 = tail call <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) null, i32 %arg0, i32 0, i32 0)
+ store <4 x i32> %i24, ptr addrspace(3) zeroinitializer, align 16
+ %p18 = getelementptr %f8, ptr addrspace(3) @shared, i32 %arg2
+ store <4 x i32> %i25, ptr addrspace(3) %p18, align 16
+ %p17 = getelementptr %f8, ptr addrspace(3) %p18, i32 512
+ %i47 = or i32 %x5, 1
+ %p16 = getelementptr %f8, ptr addrspace(3) %p17, i32 %i47
+ store <4 x i32> %i26, ptr addrspace(3) %p16, align 16
+ %i48 = load <4 x i8>, ptr addrspace(3) %p15, align 8
+ %bc16 = bitcast <4 x i32> %i20 to <2 x i64>
+ %be21 = extractelement <2 x i64> %bc16, i64 0
+ %bc11 = bitcast <4 x i32> %i22 to <2 x i64>
+ %be22 = extractelement <2 x i64> %bc11, i64 0
+ %as6 = shufflevector <4 x i8> zeroinitializer, <4 x i8> %i27, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %ac7 = bitcast <16 x i8> %as6 to <2 x i64>
+ %ae6 = extractelement <2 x i64> %ac7, i64 0
+ %i49 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
+ %i50 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae6, i64 0, <4 x float> %i49, i32 0, i32 0, i32 0)
+ %as4 = shufflevector <4 x i8> %i30, <4 x i8> zeroinitializer, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %ac6 = bitcast <16 x i8> %as4 to <2 x i64>
+ %ae7 = extractelement <2 x i64> %ac6, i64 1
+ %i51 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 72340172838076673, i64 0, <4 x float> %i50, i32 0, i32 0, i32 0)
+ %i52 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i51, i32 0, i32 0, i32 0)
+ %i53 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae7, i64 0, <4 x float> %i52, i32 0, i32 0, i32 0)
+ %i54 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i53, i32 0, i32 0, i32 0)
+ %i55 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i54, i32 0, i32 0, i32 0)
+ %i56 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be21, <4 x float> %i55, i32 0, i32 0, i32 0)
+ %bc4 = bitcast <4 x i32> %i21 to <2 x i64>
+ %be29 = extractelement <2 x i64> %bc4, i64 1
+ %be14 = extractelement <2 x i64> %bc4, i64 0
+ %i57 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be21, <4 x float> %i56, i32 0, i32 0, i32 0)
+ %i58 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be14, <4 x float> %i57, i32 0, i32 0, i32 0)
+ %i59 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be29, <4 x float> %i58, i32 0, i32 0, i32 0)
+ %i60 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i59, i32 0, i32 0, i32 0)
+ %i61 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i60, i32 0, i32 0, i32 0)
+ %as8 = shufflevector <4 x i8> zeroinitializer, <4 x i8> %arg9, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %ac9 = bitcast <16 x i8> %as8 to <2 x i64>
+ %as13 = shufflevector <4 x i8> %arg8, <4 x i8> zeroinitializer, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %ac18 = bitcast <16 x i8> %as13 to <2 x i64>
+ %ae18 = extractelement <2 x i64> %ac18, i64 1
+ %i62 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i19, i32 0, i32 0, i32 0)
+ %ae9 = extractelement <2 x i64> %ac9, i64 0
+ %i63 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae18, i64 0, <4 x float> %i62, i32 0, i32 0, i32 0)
+ %i64 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae9, i64 0, <4 x float> %i63, i32 0, i32 0, i32 0)
+ %i65 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i64, i32 0, i32 0, i32 0)
+ %as5 = shufflevector <4 x i8> zeroinitializer, <4 x i8> %i29, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %ac8 = bitcast <16 x i8> %as5 to <2 x i64>
+ %ae8 = extractelement <2 x i64> %ac8, i64 0
+ %i66 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i65, i32 0, i32 0, i32 0)
+ %i67 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i66, i32 0, i32 0, i32 0)
+ %i68 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae8, i64 0, <4 x float> %i67, i32 0, i32 0, i32 0)
+ %i69 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i68, i32 0, i32 0, i32 0)
+ %i70 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i69, i32 0, i32 0, i32 0)
+ %i71 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i70, i32 0, i32 0, i32 0)
+ %i72 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i71, i32 0, i32 0, i32 0)
+ %i73 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 0, <4 x float> %i72, i32 0, i32 0, i32 0)
+ %bc13 = bitcast <4 x i32> %i23 to <2 x i64>
+ %i74 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be22, <4 x float> %i73, i32 0, i32 0, i32 0)
+ %be20 = extractelement <2 x i64> %bc13, i64 0
+ %be26 = extractelement <2 x i64> %bc11, i64 1
+ %i75 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be26, <4 x float> %i74, i32 0, i32 0, i32 0)
+ %i76 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be20, <4 x float> %i75, i32 0, i32 0, i32 0)
+ %i77 = load <4 x i8>, ptr addrspace(3) zeroinitializer, align 8
+ %i78 = getelementptr i8, ptr addrspace(3) @shared, i32 %i28
+ %p10 = getelementptr %f8, ptr addrspace(3) %i78, i32 %arg1
+ %i79 = load <4 x i8>, ptr addrspace(3) %p10, align 4
+ %i80 = getelementptr i8, ptr addrspace(3) @shared, i32 %n4
+ %p11 = getelementptr %f8, ptr addrspace(3) %i80, i32 %arg1
+ %i81 = getelementptr i8, ptr addrspace(3) %p11, i32 4
+ %i82 = load <4 x i8>, ptr addrspace(3) %i81, align 4
+ %i83 = load <4 x i8>, ptr addrspace(3) %p11, align 8
+ %p4 = getelementptr %f8, ptr addrspace(3) %arg, i32 %m3
+ %i84 = load <4 x i8>, ptr addrspace(3) %p4, align 4
+ %i85 = getelementptr i8, ptr addrspace(3) %lds, i32 %n5
+ %p8 = getelementptr %f8, ptr addrspace(3) %i85, i32 %arg1
+ %i86 = load <4 x i8>, ptr addrspace(3) %p8, align 4
+ %i87 = load <4 x i8>, ptr addrspace(3) %i85, align 8
+ %i88 = load <4 x i8>, ptr addrspace(3) %i11, align 4
+ %p5 = getelementptr %f8, ptr addrspace(3) %lds, i32 %m3
+ %i89 = load <4 x i8>, ptr addrspace(3) %p5, align 4
+ %p13 = getelementptr i8, ptr addrspace(3) inttoptr (i32 8192 to ptr addrspace(3)), i32 %i13
+ %i90 = load <4 x i8>, ptr addrspace(3) %p13, align 4
+ %p2 = getelementptr i8, ptr addrspace(3) %arg4, i32 %n3
+ %i91 = load <4 x i8>, ptr addrspace(3) %p2, align 16
+ %p9 = getelementptr i8, ptr addrspace(3) inttoptr (i32 8192 to ptr addrspace(3)), i32 %x3
+ %i92 = load <4 x i8>, ptr addrspace(3) %p9, align 16
+ tail call void @llvm.amdgcn.sched.barrier(i32 0)
+ %bc = bitcast <4 x i32> %i32 to <2 x i64>
+ %be = extractelement <2 x i64> %bc, i64 1
+ %as14 = shufflevector <4 x i8> zeroinitializer, <4 x i8> %i79, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %bc18 = bitcast <4 x i32> %i35 to <2 x i64>
+ %be30 = extractelement <2 x i64> %bc18, i64 0
+ %ac17 = bitcast <16 x i8> %as14 to <2 x i64>
+ %ae17 = extractelement <2 x i64> %ac17, i64 1
+ %as16 = shufflevector <4 x i8> zeroinitializer, <4 x i8> %i82, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %as10 = shufflevector <4 x i8> %i83, <4 x i8> %i48, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %ac15 = bitcast <16 x i8> %as16 to <2 x i64>
+ %ae13 = extractelement <2 x i64> %ac15, i64 0
+ %bc15 = bitcast <4 x i32> %i36 to <2 x i64>
+ %be8 = extractelement <2 x i64> %bc15, i64 0
+ %ac10 = bitcast <16 x i8> %as10 to <2 x i64>
+ %ae10 = extractelement <2 x i64> %ac10, i64 1
+ %as9 = shufflevector <4 x i8> zeroinitializer, <4 x i8> %i84, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %bc7 = bitcast <4 x i32> %i38 to <2 x i64>
+ %be7 = extractelement <2 x i64> %bc7, i64 0
+ %ac16 = bitcast <16 x i8> %as9 to <2 x i64>
+ %ae11 = extractelement <2 x i64> %ac16, i64 1
+ %as11 = shufflevector <4 x i8> zeroinitializer, <4 x i8> %i86, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %as7 = shufflevector <4 x i8> %i87, <4 x i8> %i77, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %ac11 = bitcast <16 x i8> %as11 to <2 x i64>
+ %ae15 = extractelement <2 x i64> %ac11, i64 0
+ %bc6 = bitcast <4 x i32> %i39 to <2 x i64>
+ %be17 = extractelement <2 x i64> %bc6, i64 0
+ %ac14 = bitcast <16 x i8> %as7 to <2 x i64>
+ %ae12 = extractelement <2 x i64> %ac14, i64 1
+ %as12 = shufflevector <4 x i8> zeroinitializer, <4 x i8> %i88, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %ac13 = bitcast <16 x i8> %as12 to <2 x i64>
+ %ae16 = extractelement <2 x i64> %ac13, i64 0
+ %bc14 = bitcast <4 x i32> %i42 to <2 x i64>
+ %be28 = extractelement <2 x i64> %bc14, i64 0
+ %as15 = shufflevector <4 x i8> zeroinitializer, <4 x i8> %i89, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %ac12 = bitcast <16 x i8> %as15 to <2 x i64>
+ %ae14 = extractelement <2 x i64> %ac12, i64 0
+ %bc12 = bitcast <4 x i32> %i43 to <2 x i64>
+ %be19 = extractelement <2 x i64> %bc12, i64 0
+ %bc2 = bitcast <4 x i32> %i46 to <2 x i64>
+ %be15 = extractelement <2 x i64> %bc2, i64 1
+ %i93 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be15, <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
+ %bc17 = bitcast <4 x i32> %i33 to <2 x i64>
+ %be32 = extractelement <2 x i64> %bc17, i64 0
+ %bc19 = bitcast <4 x i32> %i34 to <2 x i64>
+ %be33 = extractelement <2 x i64> %bc19, i64 0
+ %i94 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae17, i64 %be33, <4 x float> zeroinitializer, i32 0, i32 0, i32 0)
+ %i95 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae13, i64 0, <4 x float> %i94, i32 0, i32 0, i32 0)
+ %i96 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae10, i64 0, <4 x float> %i95, i32 0, i32 0, i32 0)
+ %bc5 = bitcast <4 x i32> %i37 to <2 x i64>
+ %be10 = extractelement <2 x i64> %bc5, i64 0
+ %i97 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be10, <4 x float> %i96, i32 0, i32 0, i32 0)
+ %be12 = extractelement <2 x i64> %bc5, i64 1
+ %i98 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae11, i64 %be12, <4 x float> %i97, i32 0, i32 0, i32 0)
+ %bc10 = bitcast <4 x i32> %i40 to <2 x i64>
+ %be16 = extractelement <2 x i64> %bc10, i64 0
+ %i99 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae15, i64 %be16, <4 x float> %i98, i32 0, i32 0, i32 0)
+ %be6 = extractelement <2 x i64> %bc10, i64 1
+ %i100 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae12, i64 %be6, <4 x float> %i99, i32 0, i32 0, i32 0)
+ %bc8 = bitcast <4 x i32> %i41 to <2 x i64>
+ %be3 = extractelement <2 x i64> %bc8, i64 0
+ %i101 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae16, i64 %be3, <4 x float> %i100, i32 0, i32 0, i32 0)
+ %be11 = extractelement <2 x i64> %bc8, i64 1
+ %i102 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be11, <4 x float> %i101, i32 0, i32 0, i32 0)
+ %bc9 = bitcast <4 x i32> %i44 to <2 x i64>
+ %i103 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae14, i64 0, <4 x float> %i102, i32 0, i32 0, i32 0)
+ %be18 = extractelement <2 x i64> %bc9, i64 1
+ %i104 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be18, <4 x float> %i103, i32 0, i32 0, i32 0)
+ %bc3 = bitcast <4 x i32> %i45 to <2 x i64>
+ %be4 = extractelement <2 x i64> %bc3, i64 0
+ %i105 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be4, <4 x float> %i104, i32 0, i32 0, i32 0)
+ %be25 = extractelement <2 x i64> %bc3, i64 1
+ %i106 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be25, <4 x float> %i105, i32 0, i32 0, i32 0)
+ %be2 = extractelement <2 x i64> %bc, i64 0
+ %i107 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 72340172838076673, i64 %be2, <4 x float> %i61, i32 0, i32 0, i32 0)
+ %i108 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be, <4 x float> %i107, i32 0, i32 0, i32 0)
+ %i109 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be30, <4 x float> %i108, i32 0, i32 0, i32 0)
+ %be31 = extractelement <2 x i64> %bc18, i64 1
+ %i110 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be31, <4 x float> %i109, i32 0, i32 0, i32 0)
+ %i111 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be8, <4 x float> %i110, i32 0, i32 0, i32 0)
+ %be23 = extractelement <2 x i64> %bc15, i64 1
+ %i112 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be23, <4 x float> %i111, i32 0, i32 0, i32 0)
+ %i113 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be7, <4 x float> %i112, i32 0, i32 0, i32 0)
+ %be5 = extractelement <2 x i64> %bc7, i64 1
+ %i114 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be5, <4 x float> %i113, i32 0, i32 0, i32 0)
+ %as = shufflevector <4 x i8> zeroinitializer, <4 x i8> %i90, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+ %i115 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be17, <4 x float> %i114, i32 0, i32 0, i32 0)
+ %ac4 = bitcast <16 x i8> %as to <2 x i64>
+ %ae4 = extractelement <2 x i64> %ac4, i64 1
+ %be13 = extractelement <2 x i64> %bc6, i64 1
+ %i116 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae4, i64 %be13, <4 x float> %i115, i32 0, i32 0, i32 0)
+ %as3 = shufflevector <4 x i8> %i91, <4 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %ac5 = bitcast <16 x i8> %as3 to <2 x i64>
+ %ae3 = extractelement <2 x i64> %ac5, i64 0
+ %i117 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be28, <4 x float> %i116, i32 0, i32 0, i32 0)
+ %be9 = extractelement <2 x i64> %bc14, i64 1
+ %i118 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be9, <4 x float> %i117, i32 0, i32 0, i32 0)
+ %as2 = shufflevector <4 x i8> %i92, <4 x i8> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+ %ac3 = bitcast <16 x i8> %as2 to <2 x i64>
+ %ae5 = extractelement <2 x i64> %ac3, i64 0
+ %be27 = extractelement <2 x i64> %bc2, i64 0
+ %i119 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae5, i64 %be19, <4 x float> %i118, i32 0, i32 0, i32 0)
+ %i120 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be27, <4 x float> %i119, i32 0, i32 0, i32 0)
+ %i121 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be32, <4 x float> %i76, i32 0, i32 0, i32 0)
+ %be24 = extractelement <2 x i64> %bc9, i64 0
+ %i122 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 %ae3, i64 0, <4 x float> %i121, i32 0, i32 0, i32 0)
+ %i123 = tail call <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64 0, i64 %be24, <4 x float> %i122, i32 0, i32 0, i32 0)
+ %n7 = and i32 %i10, 31
+ %m4 = and i32 %i31, 56
+ %m = mul i32 %n7, %arg0
+ %a6 = or i32 %m, %m4
+ %ce4 = extractelement <4 x float> %i93, i64 0
+ %m2 = and i32 %i, 48
+ %i124 = getelementptr float, ptr addrspace(3) @shared, i32 %m2
+ %p21 = getelementptr float, ptr addrspace(3) %i124, i32 %n6
+ store float %ce4, ptr addrspace(3) zeroinitializer, align 4
+ %ce3 = extractelement <4 x float> %i120, i64 0
+ store float %ce3, ptr addrspace(3) %p21, align 4
+ %ce2 = extractelement <4 x float> %i106, i64 0
+ store float %ce2, ptr addrspace(3) zeroinitializer, align 4
+ %ce = extractelement <4 x float> %i123, i64 0
+ store float %ce, ptr addrspace(3) %p15, align 4
+ %sx = sext i32 %a6 to i64
+ %p20 = getelementptr i16, ptr addrspace(1) null, i64 %sx
+ %i125 = atomicrmw fadd ptr addrspace(1) %p20, <2 x bfloat> zeroinitializer syncscope("agent") monotonic, align 4
+ ret void
+}
+
+; Function Attrs: convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none)
+declare <4 x float> @llvm.amdgcn.mfma.f32.16x16x32.fp8.fp8(i64, i64, <4 x float>, i32 immarg, i32 immarg, i32 immarg) #1
+
+; Function Attrs: nocallback nofree nosync nounwind willreturn memory(argmem: read)
+declare <4 x i32> @llvm.amdgcn.raw.ptr.buffer.load.v4i32(ptr addrspace(8) readonly captures(none), i32, i32, i32 immarg) #2
+
+; Function Attrs: nocallback nofree nounwind willreturn
+declare void @llvm.amdgcn.s.waitcnt(i32 immarg) #3
+
+; Function Attrs: convergent nocallback nofree nounwind willreturn
+declare void @llvm.amdgcn.sched.barrier(i32 immarg) #4
+
+; Function Attrs: convergent nocallback nofree nounwind willreturn
+declare void @llvm.amdgcn.sched.group.barrier(i32 immarg, i32 immarg, i32 immarg) #4
+
+; Function Attrs: nocallback nofree nosync nounwind speculatable willreturn memory(none)
+declare noundef range(i32 0, 1024) i32 @llvm.amdgcn.workitem.id.x() #5
+
+attributes #0 = { "amdgpu-agpr-alloc"="0" "amdgpu-flat-work-group-size"="1,256" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" "amdgpu-waves-per-eu"="2" "target-cpu"="gfx942" }
+attributes #1 = { convergent nocallback nocreateundeforpoison nofree nosync nounwind willreturn memory(none) "target-cpu"="gfx942" }
+attributes #2 = { nocallback nofree nosync nounwind willreturn memory(argmem: read) "target-cpu"="gfx942" }
+attributes #3 = { nocallback nofree nounwind willreturn "target-cpu"="gfx942" }
+attributes #4 = { convergent nocallback nofree nounwind willreturn "target-cpu"="gfx942" }
+attributes #5 = { nocallback nofree nosync nounwind speculatable willreturn memory(none) "target-cpu"="gfx942" }
diff --git a/llvm/test/CodeGen/AMDGPU/mul.ll b/llvm/test/CodeGen/AMDGPU/mul.ll
index d10e4511a5c6a..daaf688cb0193 100644
--- a/llvm/test/CodeGen/AMDGPU/mul.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul.ll
@@ -751,71 +751,67 @@ entry:
define amdgpu_kernel void @mul64_sext_c(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: mul64_sext_c:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: v_mov_b32_e32 v0, 0x50
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mul_hi_i32 v1, s6, v0
-; SI-NEXT: s_mulk_i32 s6, 0x50
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: v_mul_hi_i32 v1, s2, v0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mul_i32 s0, s2, 0x50
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: mul64_sext_c:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 0x50
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mad_i64_i32 v[0:1], s[2:3], s2, v0, 0
; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: v_mad_i64_i32 v[0:1], s[4:5], s2, v0, 0
; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: s_nop 2
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: mul64_sext_c:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_hi_i32 s4, s6, 0x50
-; GFX9-NEXT: s_mulk_i32 s6, 0x50
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_mul_hi_i32 s0, s2, 0x50
+; GFX9-NEXT: s_mul_i32 s1, s2, 0x50
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: mul64_sext_c:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mul_i32 s3, s2, 0x50
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000
+; GFX10-NEXT: s_mul_i32 s4, s2, 0x50
; GFX10-NEXT: s_mul_hi_i32 s2, s2, 0x50
-; GFX10-NEXT: v_mov_b32_e32 v0, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000
; GFX10-NEXT: s_mov_b32 s2, -1
; GFX10-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: mul64_sext_c:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mul_i32 s3, s2, 0x50
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_mul_i32 s4, s2, 0x50
; GFX11-NEXT: s_mul_hi_i32 s2, s2, 0x50
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: s_mov_b32 s2, -1
; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
; GFX11-NEXT: s_endpgm
@@ -882,71 +878,67 @@ entry:
define amdgpu_kernel void @mul64_zext_c(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: mul64_zext_c:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: v_mov_b32_e32 v0, 0x50
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mul_hi_u32 v1, s6, v0
-; SI-NEXT: s_mulk_i32 s6, 0x50
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: v_mul_hi_u32 v1, s2, v0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mul_i32 s0, s2, 0x50
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: mul64_zext_c:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 0x50
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mad_u64_u32 v[0:1], s[2:3], s2, v0, 0
; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: v_mad_u64_u32 v[0:1], s[4:5], s2, v0, 0
; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: s_nop 2
; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: mul64_zext_c:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mul_hi_u32 s4, s6, 0x50
-; GFX9-NEXT: s_mulk_i32 s6, 0x50
-; GFX9-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-NEXT: v_mov_b32_e32 v1, s4
-; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_mul_hi_u32 s0, s2, 0x50
+; GFX9-NEXT: s_mul_i32 s1, s2, 0x50
+; GFX9-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: mul64_zext_c:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mul_i32 s3, s2, 0x50
+; GFX10-NEXT: s_mov_b32 s3, 0x31016000
+; GFX10-NEXT: s_mul_i32 s4, s2, 0x50
; GFX10-NEXT: s_mul_hi_u32 s2, s2, 0x50
-; GFX10-NEXT: v_mov_b32_e32 v0, s3
+; GFX10-NEXT: v_mov_b32_e32 v0, s4
; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000
; GFX10-NEXT: s_mov_b32 s2, -1
; GFX10-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: mul64_zext_c:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mul_i32 s3, s2, 0x50
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_mul_i32 s4, s2, 0x50
; GFX11-NEXT: s_mul_hi_u32 s2, s2, 0x50
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s2
; GFX11-NEXT: s_mov_b32 s2, -1
; GFX11-NEXT: buffer_store_b64 v[0:1], off, s[0:3], 0
; GFX11-NEXT: s_endpgm
@@ -2636,33 +2628,27 @@ entry:
define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(1) %in, i32 %a, i32 %b, i32 %c) {
; SI-LABEL: mul32_in_branch:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_lg_u32 s0, 0
-; SI-NEXT: s_cbranch_scc0 .LBB15_2
-; SI-NEXT: ; %bb.1: ; %else
-; SI-NEXT: s_mul_i32 s8, s0, s1
-; SI-NEXT: s_mov_b64 s[6:7], 0
-; SI-NEXT: s_branch .LBB15_3
-; SI-NEXT: .LBB15_2:
-; SI-NEXT: s_mov_b64 s[6:7], -1
-; SI-NEXT: ; implicit-def: $sgpr8
-; SI-NEXT: .LBB15_3: ; %Flow
+; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
-; SI-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 vcc, vcc
-; SI-NEXT: s_cbranch_vccnz .LBB15_5
-; SI-NEXT: ; %bb.4: ; %if
+; SI-NEXT: s_cmp_lg_u32 s6, 0
+; SI-NEXT: s_cbranch_scc0 .LBB15_3
+; SI-NEXT: ; %bb.1: ; %else
+; SI-NEXT: s_mul_i32 s6, s6, s7
+; SI-NEXT: s_cbranch_execnz .LBB15_4
+; SI-NEXT: .LBB15_2: ; %if
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_mov_b32 s4, s2
; SI-NEXT: s_mov_b32 s5, s3
; SI-NEXT: buffer_load_dword v0, off, s[4:7], 0
-; SI-NEXT: s_branch .LBB15_6
-; SI-NEXT: .LBB15_5:
-; SI-NEXT: v_mov_b32_e32 v0, s8
-; SI-NEXT: .LBB15_6: ; %endif
+; SI-NEXT: s_branch .LBB15_5
+; SI-NEXT: .LBB15_3:
+; SI-NEXT: ; implicit-def: $sgpr6
+; SI-NEXT: s_branch .LBB15_2
+; SI-NEXT: .LBB15_4:
+; SI-NEXT: v_mov_b32_e32 v0, s6
+; SI-NEXT: .LBB15_5: ; %endif
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: s_waitcnt vmcnt(0)
@@ -2671,33 +2657,27 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
;
; VI-LABEL: mul32_in_branch:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_lg_u32 s0, 0
-; VI-NEXT: s_cbranch_scc0 .LBB15_2
+; VI-NEXT: s_cmp_lg_u32 s6, 0
+; VI-NEXT: s_cbranch_scc0 .LBB15_3
; VI-NEXT: ; %bb.1: ; %else
-; VI-NEXT: s_mul_i32 s8, s0, s1
-; VI-NEXT: s_mov_b64 s[6:7], 0
-; VI-NEXT: s_branch .LBB15_3
-; VI-NEXT: .LBB15_2:
-; VI-NEXT: s_mov_b64 s[6:7], -1
-; VI-NEXT: ; implicit-def: $sgpr8
-; VI-NEXT: .LBB15_3: ; %Flow
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; VI-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; VI-NEXT: s_cbranch_vccnz .LBB15_5
-; VI-NEXT: ; %bb.4: ; %if
+; VI-NEXT: s_mul_i32 s6, s6, s7
+; VI-NEXT: s_cbranch_execnz .LBB15_4
+; VI-NEXT: .LBB15_2: ; %if
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
-; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_mov_b32 s4, s2
; VI-NEXT: s_mov_b32 s5, s3
; VI-NEXT: buffer_load_dword v0, off, s[4:7], 0
-; VI-NEXT: s_branch .LBB15_6
-; VI-NEXT: .LBB15_5:
-; VI-NEXT: v_mov_b32_e32 v0, s8
-; VI-NEXT: .LBB15_6: ; %endif
-; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_branch .LBB15_5
+; VI-NEXT: .LBB15_3:
+; VI-NEXT: ; implicit-def: $sgpr6
+; VI-NEXT: s_branch .LBB15_2
+; VI-NEXT: .LBB15_4:
+; VI-NEXT: v_mov_b32_e32 v0, s6
+; VI-NEXT: .LBB15_5: ; %endif
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt vmcnt(0)
@@ -2706,33 +2686,27 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
;
; GFX9-LABEL: mul32_in_branch:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_cmp_lg_u32 s0, 0
-; GFX9-NEXT: s_cbranch_scc0 .LBB15_2
+; GFX9-NEXT: s_cmp_lg_u32 s6, 0
+; GFX9-NEXT: s_cbranch_scc0 .LBB15_3
; GFX9-NEXT: ; %bb.1: ; %else
-; GFX9-NEXT: s_mul_i32 s8, s0, s1
-; GFX9-NEXT: s_mov_b64 s[6:7], 0
-; GFX9-NEXT: s_branch .LBB15_3
-; GFX9-NEXT: .LBB15_2:
-; GFX9-NEXT: s_mov_b64 s[6:7], -1
-; GFX9-NEXT: ; implicit-def: $sgpr8
-; GFX9-NEXT: .LBB15_3: ; %Flow
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX9-NEXT: s_andn2_b64 vcc, exec, s[6:7]
-; GFX9-NEXT: s_cbranch_vccnz .LBB15_5
-; GFX9-NEXT: ; %bb.4: ; %if
+; GFX9-NEXT: s_mul_i32 s6, s6, s7
+; GFX9-NEXT: s_cbranch_execnz .LBB15_4
+; GFX9-NEXT: .LBB15_2: ; %if
; GFX9-NEXT: s_mov_b32 s7, 0xf000
; GFX9-NEXT: s_mov_b32 s6, -1
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_mov_b32 s4, s2
; GFX9-NEXT: s_mov_b32 s5, s3
; GFX9-NEXT: buffer_load_dword v0, off, s[4:7], 0
-; GFX9-NEXT: s_branch .LBB15_6
-; GFX9-NEXT: .LBB15_5:
-; GFX9-NEXT: v_mov_b32_e32 v0, s8
-; GFX9-NEXT: .LBB15_6: ; %endif
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: s_branch .LBB15_5
+; GFX9-NEXT: .LBB15_3:
+; GFX9-NEXT: ; implicit-def: $sgpr6
+; GFX9-NEXT: s_branch .LBB15_2
+; GFX9-NEXT: .LBB15_4:
+; GFX9-NEXT: v_mov_b32_e32 v0, s6
+; GFX9-NEXT: .LBB15_5: ; %endif
; GFX9-NEXT: s_mov_b32 s3, 0xf000
; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt vmcnt(0)
@@ -2741,33 +2715,30 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
;
; GFX10-LABEL: mul32_in_branch:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX10-NEXT: s_mov_b32 s6, 0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_cmp_lg_u32 s0, 0
-; GFX10-NEXT: s_cbranch_scc0 .LBB15_2
+; GFX10-NEXT: s_cmp_lg_u32 s6, 0
+; GFX10-NEXT: s_cbranch_scc0 .LBB15_3
; GFX10-NEXT: ; %bb.1: ; %else
-; GFX10-NEXT: s_mul_i32 s7, s0, s1
-; GFX10-NEXT: s_branch .LBB15_3
-; GFX10-NEXT: .LBB15_2:
-; GFX10-NEXT: s_mov_b32 s6, -1
-; GFX10-NEXT: ; implicit-def: $sgpr7
-; GFX10-NEXT: .LBB15_3: ; %Flow
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
-; GFX10-NEXT: s_andn2_b32 vcc_lo, exec_lo, s6
-; GFX10-NEXT: s_cbranch_vccnz .LBB15_5
-; GFX10-NEXT: ; %bb.4: ; %if
+; GFX10-NEXT: s_mul_i32 s5, s6, s7
+; GFX10-NEXT: s_andn2_b32 vcc_lo, exec_lo, s4
+; GFX10-NEXT: s_cbranch_vccnz .LBB15_4
+; GFX10-NEXT: .LBB15_2: ; %if
; GFX10-NEXT: s_mov_b32 s7, 0x31016000
; GFX10-NEXT: s_mov_b32 s6, -1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_mov_b32 s4, s2
; GFX10-NEXT: s_mov_b32 s5, s3
; GFX10-NEXT: buffer_load_dword v0, off, s[4:7], 0
-; GFX10-NEXT: s_branch .LBB15_6
-; GFX10-NEXT: .LBB15_5:
-; GFX10-NEXT: v_mov_b32_e32 v0, s7
-; GFX10-NEXT: .LBB15_6: ; %endif
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: s_branch .LBB15_5
+; GFX10-NEXT: .LBB15_3:
+; GFX10-NEXT: ; implicit-def: $sgpr5
+; GFX10-NEXT: s_branch .LBB15_2
+; GFX10-NEXT: .LBB15_4:
+; GFX10-NEXT: v_mov_b32_e32 v0, s5
+; GFX10-NEXT: .LBB15_5: ; %endif
; GFX10-NEXT: s_mov_b32 s3, 0x31016000
; GFX10-NEXT: s_mov_b32 s2, -1
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -2776,33 +2747,30 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
;
; GFX11-LABEL: mul32_in_branch:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX11-NEXT: s_mov_b32 s6, 0
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_cmp_lg_u32 s0, 0
-; GFX11-NEXT: s_cbranch_scc0 .LBB15_2
+; GFX11-NEXT: s_cmp_lg_u32 s6, 0
+; GFX11-NEXT: s_cbranch_scc0 .LBB15_3
; GFX11-NEXT: ; %bb.1: ; %else
-; GFX11-NEXT: s_mul_i32 s7, s0, s1
-; GFX11-NEXT: s_branch .LBB15_3
-; GFX11-NEXT: .LBB15_2:
-; GFX11-NEXT: s_mov_b32 s6, -1
-; GFX11-NEXT: ; implicit-def: $sgpr7
-; GFX11-NEXT: .LBB15_3: ; %Flow
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX11-NEXT: s_cbranch_vccnz .LBB15_5
-; GFX11-NEXT: ; %bb.4: ; %if
+; GFX11-NEXT: s_mul_i32 s5, s6, s7
+; GFX11-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX11-NEXT: s_cbranch_vccnz .LBB15_4
+; GFX11-NEXT: .LBB15_2: ; %if
; GFX11-NEXT: s_mov_b32 s7, 0x31016000
; GFX11-NEXT: s_mov_b32 s6, -1
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_mov_b32 s4, s2
; GFX11-NEXT: s_mov_b32 s5, s3
; GFX11-NEXT: buffer_load_b32 v0, off, s[4:7], 0
-; GFX11-NEXT: s_branch .LBB15_6
-; GFX11-NEXT: .LBB15_5:
-; GFX11-NEXT: v_mov_b32_e32 v0, s7
-; GFX11-NEXT: .LBB15_6: ; %endif
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_branch .LBB15_5
+; GFX11-NEXT: .LBB15_3:
+; GFX11-NEXT: ; implicit-def: $sgpr5
+; GFX11-NEXT: s_branch .LBB15_2
+; GFX11-NEXT: .LBB15_4:
+; GFX11-NEXT: v_mov_b32_e32 v0, s5
+; GFX11-NEXT: .LBB15_5: ; %endif
; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: s_mov_b32 s2, -1
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -2811,33 +2779,30 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
;
; GFX12-LABEL: mul32_in_branch:
; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX12-NEXT: s_mov_b32 s6, 0
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_cmp_lg_u32 s0, 0
-; GFX12-NEXT: s_cbranch_scc0 .LBB15_2
+; GFX12-NEXT: s_cmp_lg_u32 s6, 0
+; GFX12-NEXT: s_cbranch_scc0 .LBB15_3
; GFX12-NEXT: ; %bb.1: ; %else
-; GFX12-NEXT: s_mul_i32 s7, s0, s1
-; GFX12-NEXT: s_branch .LBB15_3
-; GFX12-NEXT: .LBB15_2:
-; GFX12-NEXT: s_mov_b32 s6, -1
-; GFX12-NEXT: ; implicit-def: $sgpr7
-; GFX12-NEXT: .LBB15_3: ; %Flow
-; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX12-NEXT: s_cbranch_vccnz .LBB15_5
-; GFX12-NEXT: ; %bb.4: ; %if
+; GFX12-NEXT: s_mul_i32 s5, s6, s7
+; GFX12-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX12-NEXT: s_cbranch_vccnz .LBB15_4
+; GFX12-NEXT: .LBB15_2: ; %if
; GFX12-NEXT: s_mov_b32 s7, 0x31016000
; GFX12-NEXT: s_mov_b32 s6, -1
-; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_mov_b32 s4, s2
; GFX12-NEXT: s_mov_b32 s5, s3
; GFX12-NEXT: buffer_load_b32 v0, off, s[4:7], null
-; GFX12-NEXT: s_branch .LBB15_6
-; GFX12-NEXT: .LBB15_5:
-; GFX12-NEXT: v_mov_b32_e32 v0, s7
-; GFX12-NEXT: .LBB15_6: ; %endif
-; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_branch .LBB15_5
+; GFX12-NEXT: .LBB15_3:
+; GFX12-NEXT: ; implicit-def: $sgpr5
+; GFX12-NEXT: s_branch .LBB15_2
+; GFX12-NEXT: .LBB15_4:
+; GFX12-NEXT: v_mov_b32_e32 v0, s5
+; GFX12-NEXT: .LBB15_5: ; %endif
; GFX12-NEXT: s_mov_b32 s3, 0x31016000
; GFX12-NEXT: s_mov_b32 s2, -1
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -2847,33 +2812,31 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
; GFX1250-LABEL: mul32_in_branch:
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
-; GFX1250-NEXT: s_mov_b32 s6, 0
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_mov_b32 s4, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_cmp_lg_u32 s0, 0
-; GFX1250-NEXT: s_cbranch_scc0 .LBB15_2
+; GFX1250-NEXT: s_cmp_lg_u32 s6, 0
+; GFX1250-NEXT: s_cbranch_scc0 .LBB15_3
; GFX1250-NEXT: ; %bb.1: ; %else
-; GFX1250-NEXT: s_mul_i32 s7, s0, s1
-; GFX1250-NEXT: s_branch .LBB15_3
-; GFX1250-NEXT: .LBB15_2:
-; GFX1250-NEXT: s_mov_b32 s6, -1
-; GFX1250-NEXT: ; implicit-def: $sgpr7
-; GFX1250-NEXT: .LBB15_3: ; %Flow
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX1250-NEXT: s_cbranch_vccnz .LBB15_5
-; GFX1250-NEXT: ; %bb.4: ; %if
+; GFX1250-NEXT: s_mul_i32 s5, s6, s7
+; GFX1250-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX1250-NEXT: s_cbranch_vccnz .LBB15_4
+; GFX1250-NEXT: .LBB15_2: ; %if
; GFX1250-NEXT: s_mov_b32 s7, 0x31016000
; GFX1250-NEXT: s_mov_b32 s6, -1
-; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_mov_b32 s4, s2
; GFX1250-NEXT: s_mov_b32 s5, s3
; GFX1250-NEXT: buffer_load_b32 v0, off, s[4:7], null
-; GFX1250-NEXT: s_branch .LBB15_6
-; GFX1250-NEXT: .LBB15_5:
-; GFX1250-NEXT: v_mov_b32_e32 v0, s7
-; GFX1250-NEXT: .LBB15_6: ; %endif
-; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_branch .LBB15_5
+; GFX1250-NEXT: .LBB15_3:
+; GFX1250-NEXT: ; implicit-def: $sgpr5
+; GFX1250-NEXT: s_branch .LBB15_2
+; GFX1250-NEXT: .LBB15_4:
+; GFX1250-NEXT: v_mov_b32_e32 v0, s5
+; GFX1250-NEXT: .LBB15_5: ; %endif
; GFX1250-NEXT: s_mov_b32 s3, 0x31016000
; GFX1250-NEXT: s_mov_b32 s2, -1
; GFX1250-NEXT: s_wait_loadcnt 0x0
@@ -2882,33 +2845,30 @@ define amdgpu_kernel void @mul32_in_branch(ptr addrspace(1) %out, ptr addrspace(
;
; GFX13-LABEL: mul32_in_branch:
; GFX13: ; %bb.0: ; %entry
-; GFX13-NEXT: s_load_b64 s[0:1], s[4:5], 0x34 nv
-; GFX13-NEXT: s_mov_b32 s6, 0
+; GFX13-NEXT: s_clause 0x1
+; GFX13-NEXT: s_load_b64 s[6:7], s[4:5], 0x34 nv
+; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX13-NEXT: s_mov_b32 s4, 0
; GFX13-NEXT: s_wait_kmcnt 0x0
-; GFX13-NEXT: s_cmp_lg_u32 s0, 0
-; GFX13-NEXT: s_cbranch_scc0 .LBB15_2
+; GFX13-NEXT: s_cmp_lg_u32 s6, 0
+; GFX13-NEXT: s_cbranch_scc0 .LBB15_3
; GFX13-NEXT: ; %bb.1: ; %else
-; GFX13-NEXT: s_mul_i32 s7, s0, s1
-; GFX13-NEXT: s_branch .LBB15_3
-; GFX13-NEXT: .LBB15_2:
-; GFX13-NEXT: s_mov_b32 s6, -1
-; GFX13-NEXT: ; implicit-def: $sgpr7
-; GFX13-NEXT: .LBB15_3: ; %Flow
-; GFX13-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX13-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s6
-; GFX13-NEXT: s_cbranch_vccnz .LBB15_5
-; GFX13-NEXT: ; %bb.4: ; %if
+; GFX13-NEXT: s_mul_i32 s5, s6, s7
+; GFX13-NEXT: s_and_not1_b32 vcc_lo, exec_lo, s4
+; GFX13-NEXT: s_cbranch_vccnz .LBB15_4
+; GFX13-NEXT: .LBB15_2: ; %if
; GFX13-NEXT: s_mov_b32 s7, 0x31016000
; GFX13-NEXT: s_mov_b32 s6, -1
-; GFX13-NEXT: s_wait_kmcnt 0x0
; GFX13-NEXT: s_mov_b32 s4, s2
; GFX13-NEXT: s_mov_b32 s5, s3
; GFX13-NEXT: buffer_load_b32 v0, off, s[4:7], null
-; GFX13-NEXT: s_branch .LBB15_6
-; GFX13-NEXT: .LBB15_5:
-; GFX13-NEXT: v_mov_b32_e32 v0, s7
-; GFX13-NEXT: .LBB15_6: ; %endif
-; GFX13-NEXT: s_wait_kmcnt 0x0
+; GFX13-NEXT: s_branch .LBB15_5
+; GFX13-NEXT: .LBB15_3:
+; GFX13-NEXT: ; implicit-def: $sgpr5
+; GFX13-NEXT: s_branch .LBB15_2
+; GFX13-NEXT: .LBB15_4:
+; GFX13-NEXT: v_mov_b32_e32 v0, s5
+; GFX13-NEXT: .LBB15_5: ; %endif
; GFX13-NEXT: s_mov_b32 s3, 0x31016000
; GFX13-NEXT: s_mov_b32 s2, -1
; GFX13-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/mul_int24.ll b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
index 10d4eb029ee35..6e2dc46270f6a 100644
--- a/llvm/test/CodeGen/AMDGPU/mul_int24.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul_int24.ll
@@ -10,43 +10,46 @@ define amdgpu_kernel void @test_smul24_i32(ptr addrspace(1) %out, i32 %a, i32 %b
; SI-LABEL: test_smul24_i32:
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_bfe_i32 s2, s4, 0x180000
-; SI-NEXT: s_bfe_i32 s4, s5, 0x180000
-; SI-NEXT: s_mul_i32 s4, s2, s4
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_bfe_i32 s2, s2, 0x180000
+; SI-NEXT: s_bfe_i32 s3, s3, 0x180000
+; SI-NEXT: s_mul_i32 s2, s2, s3
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_smul24_i32:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: s_bfe_i32 s4, s4, 0x180000
-; VI-NEXT: s_bfe_i32 s5, s5, 0x180000
-; VI-NEXT: s_mul_i32 s4, s4, s5
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_bfe_i32 s0, s2, 0x180000
+; VI-NEXT: s_bfe_i32 s1, s3, 0x180000
+; VI-NEXT: s_mul_i32 s0, s0, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: test_smul24_i32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX9-NEXT: s_bfe_i32 s4, s4, 0x180000
-; GFX9-NEXT: s_bfe_i32 s5, s5, 0x180000
-; GFX9-NEXT: s_mul_i32 s4, s4, s5
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_bfe_i32 s0, s2, 0x180000
+; GFX9-NEXT: s_bfe_i32 s1, s3, 0x180000
+; GFX9-NEXT: s_mul_i32 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: test_smul24_i32:
@@ -124,15 +127,16 @@ define amdgpu_kernel void @test_smulhi24_i64(ptr addrspace(1) %out, i32 %a, i32
; GFX9-LABEL: test_smulhi24_i64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX9-NEXT: s_bfe_i32 s4, s4, 0x180000
-; GFX9-NEXT: s_bfe_i32 s5, s5, 0x180000
-; GFX9-NEXT: s_mul_hi_i32 s4, s4, s5
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_bfe_i32 s0, s2, 0x180000
+; GFX9-NEXT: s_bfe_i32 s1, s3, 0x180000
+; GFX9-NEXT: s_mul_hi_i32 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: test_smulhi24_i64:
@@ -372,44 +376,47 @@ define amdgpu_kernel void @test_smul24_i64(ptr addrspace(1) %out, [8 x i32], i32
define amdgpu_kernel void @test_smul24_i64_square(ptr addrspace(1) %out, i32 %a, i32 %b) #0 {
; SI-LABEL: test_smul24_i64_square:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_bfe_i32 s4, s6, 0x180000
-; SI-NEXT: s_mul_i32 s5, s4, s4
-; SI-NEXT: v_mul_hi_i32_i24_e64 v1, s4, s4
-; SI-NEXT: v_mov_b32_e32 v0, s5
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_bfe_i32 s0, s2, 0x180000
+; SI-NEXT: s_mul_i32 s1, s0, s0
+; SI-NEXT: v_mul_hi_i32_i24_e64 v1, s0, s0
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_smul24_i64_square:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfe_i32 s4, s6, 0x180000
-; VI-NEXT: v_mul_hi_i32_i24_e64 v1, s4, s4
-; VI-NEXT: v_mul_i32_i24_e64 v0, s4, s4
-; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_bfe_i32 s0, s2, 0x180000
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mul_hi_i32_i24_e64 v1, s0, s0
+; VI-NEXT: v_mul_i32_i24_e64 v0, s0, s0
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: test_smul24_i64_square:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s4, s6, 0x180000
-; GFX9-NEXT: s_mul_hi_i32 s5, s4, s4
-; GFX9-NEXT: s_mul_i32 s4, s4, s4
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: v_mov_b32_e32 v1, s5
-; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_bfe_i32 s0, s2, 0x180000
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_mul_hi_i32 s1, s0, s0
+; GFX9-NEXT: s_mul_i32 s0, s0, s0
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: test_smul24_i64_square:
@@ -659,68 +666,65 @@ entry:
define amdgpu_kernel void @simplify_i24_crash(ptr addrspace(1) %out, i32 %arg0, <2 x i32> %arg1, <2 x i32> %arg2) {
; SI-LABEL: simplify_i24_crash:
; SI: ; %bb.0: ; %bb
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cmp_lg_u32 s2, 0
; SI-NEXT: s_cbranch_scc0 .LBB8_2
; SI-NEXT: ; %bb.1: ; %bb7
; SI-NEXT: s_endpgm
; SI-NEXT: .LBB8_2: ; %bb11
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xd
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_bfe_i32 s0, s0, 0x180000
-; SI-NEXT: s_bfe_i32 s1, s2, 0x180000
-; SI-NEXT: s_mul_i32 s0, s0, s1
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s0
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: s_bfe_i32 s2, s4, 0x180000
+; SI-NEXT: s_bfe_i32 s4, s6, 0x180000
+; SI-NEXT: s_mul_i32 s4, s2, s4
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mov_b32_e32 v0, s4
+; SI-NEXT: v_mov_b32_e32 v1, s4
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: simplify_i24_crash:
; VI: ; %bb.0: ; %bb
-; VI-NEXT: s_load_dword s0, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cmp_lg_u32 s2, 0
; VI-NEXT: s_cbranch_scc0 .LBB8_2
; VI-NEXT: ; %bb.1: ; %bb7
; VI-NEXT: s_endpgm
; VI-NEXT: .LBB8_2: ; %bb11
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s7, 0xf000
-; VI-NEXT: s_mov_b32 s6, -1
+; VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x34
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfe_i32 s0, s0, 0x180000
-; VI-NEXT: s_bfe_i32 s1, s2, 0x180000
-; VI-NEXT: s_mul_i32 s0, s0, s1
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s0
-; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; VI-NEXT: s_bfe_i32 s4, s4, 0x180000
+; VI-NEXT: s_bfe_i32 s5, s6, 0x180000
+; VI-NEXT: s_mul_i32 s4, s4, s5
+; VI-NEXT: v_mov_b32_e32 v0, s4
+; VI-NEXT: v_mov_b32_e32 v1, s4
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: simplify_i24_crash:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_cmp_lg_u32 s0, 0
+; GFX9-NEXT: s_cmp_lg_u32 s2, 0
; GFX9-NEXT: s_cbranch_scc0 .LBB8_2
; GFX9-NEXT: ; %bb.1: ; %bb7
; GFX9-NEXT: s_endpgm
; GFX9-NEXT: .LBB8_2: ; %bb11
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x34
-; GFX9-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s11, 0xf000
-; GFX9-NEXT: s_mov_b32 s10, -1
+; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x34
+; GFX9-NEXT: s_mov_b32 s3, 0xf000
+; GFX9-NEXT: s_mov_b32 s2, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_bfe_i32 s0, s0, 0x180000
-; GFX9-NEXT: s_bfe_i32 s1, s2, 0x180000
-; GFX9-NEXT: s_mul_i32 s0, s0, s1
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
+; GFX9-NEXT: s_bfe_i32 s4, s8, 0x180000
+; GFX9-NEXT: s_bfe_i32 s5, s10, 0x180000
+; GFX9-NEXT: s_mul_i32 s4, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
+; GFX9-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; GFX9-NEXT: s_endpgm
;
; EG-LABEL: simplify_i24_crash:
diff --git a/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll b/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll
index 1165401a93af8..6b3234fca8374 100644
--- a/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll
+++ b/llvm/test/CodeGen/AMDGPU/mul_uint24-amdgcn.ll
@@ -10,43 +10,46 @@ define amdgpu_kernel void @test_umul24_i32(ptr addrspace(1) %out, i32 %a, i32 %b
; SI-LABEL: test_umul24_i32:
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_and_b32 s2, s4, 0xffffff
-; SI-NEXT: s_and_b32 s4, s5, 0xffffff
-; SI-NEXT: s_mul_i32 s4, s2, s4
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_and_b32 s2, s2, 0xffffff
+; SI-NEXT: s_and_b32 s3, s3, 0xffffff
+; SI-NEXT: s_mul_i32 s2, s2, s3
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_umul24_i32:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: s_and_b32 s4, s4, 0xffffff
-; VI-NEXT: s_and_b32 s5, s5, 0xffffff
-; VI-NEXT: s_mul_i32 s4, s4, s5
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_and_b32 s0, s2, 0xffffff
+; VI-NEXT: s_and_b32 s1, s3, 0xffffff
+; VI-NEXT: s_mul_i32 s0, s0, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: test_umul24_i32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX9-NEXT: s_and_b32 s4, s4, 0xffffff
-; GFX9-NEXT: s_and_b32 s5, s5, 0xffffff
-; GFX9-NEXT: s_mul_i32 s4, s4, s5
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_and_b32 s0, s2, 0xffffff
+; GFX9-NEXT: s_and_b32 s1, s3, 0xffffff
+; GFX9-NEXT: s_mul_i32 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
entry:
%0 = shl i32 %a, 8
@@ -61,10 +64,9 @@ entry:
define amdgpu_kernel void @test_umul24_i16_sext(ptr addrspace(1) %out, i16 %a, i16 %b) {
; SI-LABEL: test_umul24_i16_sext:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_lshr_b32 s4, s2, 16
; SI-NEXT: s_mul_i32 s2, s2, s4
; SI-NEXT: s_sext_i32_i16 s4, s2
@@ -75,30 +77,32 @@ define amdgpu_kernel void @test_umul24_i16_sext(ptr addrspace(1) %out, i16 %a, i
;
; VI-LABEL: test_umul24_i16_sext:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s4, s6, 16
-; VI-NEXT: s_mul_i32 s6, s6, s4
-; VI-NEXT: s_sext_i32_i16 s4, s6
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_lshr_b32 s0, s2, 16
+; VI-NEXT: s_mul_i32 s0, s2, s0
+; VI-NEXT: s_sext_i32_i16 s0, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: test_umul24_i16_sext:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s6, 16
-; GFX9-NEXT: s_mul_i32 s6, s6, s4
-; GFX9-NEXT: s_sext_i32_i16 s4, s6
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_lshr_b32 s0, s2, 16
+; GFX9-NEXT: s_mul_i32 s0, s2, s0
+; GFX9-NEXT: s_sext_i32_i16 s0, s0
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
entry:
%mul = mul i16 %a, %b
@@ -183,10 +187,9 @@ define amdgpu_kernel void @test_umul24_i16_vgpr_sext(ptr addrspace(1) %out, ptr
define amdgpu_kernel void @test_umul24_i16(ptr addrspace(1) %out, i16 %a, i16 %b) {
; SI-LABEL: test_umul24_i16:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_lshr_b32 s4, s2, 16
; SI-NEXT: s_mul_i32 s2, s2, s4
; SI-NEXT: s_and_b32 s4, s2, 0xffff
@@ -197,30 +200,32 @@ define amdgpu_kernel void @test_umul24_i16(ptr addrspace(1) %out, i16 %a, i16 %b
;
; VI-LABEL: test_umul24_i16:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s4, s6, 16
-; VI-NEXT: s_mul_i32 s6, s6, s4
-; VI-NEXT: s_and_b32 s4, s6, 0xffff
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_lshr_b32 s0, s2, 16
+; VI-NEXT: s_mul_i32 s0, s2, s0
+; VI-NEXT: s_and_b32 s0, s0, 0xffff
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: test_umul24_i16:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s6, 16
-; GFX9-NEXT: s_mul_i32 s6, s6, s4
-; GFX9-NEXT: s_and_b32 s4, s6, 0xffff
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_lshr_b32 s0, s2, 16
+; GFX9-NEXT: s_mul_i32 s0, s2, s0
+; GFX9-NEXT: s_and_b32 s0, s0, 0xffff
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
entry:
%mul = mul i16 %a, %b
@@ -403,15 +408,16 @@ define amdgpu_kernel void @test_umulhi24_i32_i64(ptr addrspace(1) %out, i32 %a,
; GFX9-LABEL: test_umulhi24_i32_i64:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b32 s7, 0xf000
+; GFX9-NEXT: s_mov_b32 s6, -1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX9-NEXT: s_and_b32 s4, s4, 0xffffff
-; GFX9-NEXT: s_and_b32 s5, s5, 0xffffff
-; GFX9-NEXT: s_mul_hi_u32 s4, s4, s5
-; GFX9-NEXT: s_mov_b32 s3, 0xf000
-; GFX9-NEXT: s_mov_b32 s2, -1
-; GFX9-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX9-NEXT: s_mov_b32 s4, s0
+; GFX9-NEXT: s_mov_b32 s5, s1
+; GFX9-NEXT: s_and_b32 s0, s2, 0xffffff
+; GFX9-NEXT: s_and_b32 s1, s3, 0xffffff
+; GFX9-NEXT: s_mul_hi_u32 s0, s0, s1
+; GFX9-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX9-NEXT: s_endpgm
entry:
%a.24 = and i32 %a, 16777215
@@ -628,31 +634,33 @@ define amdgpu_kernel void @test_umulhi16_i32(ptr addrspace(1) %out, i32 %a, i32
; SI-LABEL: test_umulhi16_i32:
; SI: ; %bb.0: ; %entry
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_and_b32 s2, s4, 0xffff
-; SI-NEXT: s_and_b32 s4, s5, 0xffff
-; SI-NEXT: s_mul_i32 s2, s2, s4
-; SI-NEXT: s_lshr_b32 s4, s2, 16
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_and_b32 s2, s2, 0xffff
+; SI-NEXT: s_and_b32 s3, s3, 0xffff
+; SI-NEXT: s_mul_i32 s2, s2, s3
+; SI-NEXT: s_lshr_b32 s2, s2, 16
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: test_umulhi16_i32:
; VI: ; %bb.0: ; %entry
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: s_and_b32 s4, s4, 0xffff
-; VI-NEXT: s_and_b32 s5, s5, 0xffff
-; VI-NEXT: s_mul_i32 s4, s4, s5
-; VI-NEXT: s_lshr_b32 s4, s4, 16
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_and_b32 s0, s2, 0xffff
+; VI-NEXT: s_and_b32 s1, s3, 0xffff
+; VI-NEXT: s_mul_i32 s0, s0, s1
+; VI-NEXT: s_lshr_b32 s0, s0, 16
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: test_umulhi16_i32:
diff --git a/llvm/test/CodeGen/AMDGPU/no-shrink-extloads.ll b/llvm/test/CodeGen/AMDGPU/no-shrink-extloads.ll
index afb289bd28ecf..69e6d34e62f6f 100644
--- a/llvm/test/CodeGen/AMDGPU/no-shrink-extloads.ll
+++ b/llvm/test/CodeGen/AMDGPU/no-shrink-extloads.ll
@@ -7,7 +7,7 @@ declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
; using a buffer_load instruction.
; FUNC-LABEL: {{^}}truncate_kernarg_i32_to_i16:
-; SI: s_load_dword s
+; SI: s_load_dwordx4
; SI: buffer_store_short v
define amdgpu_kernel void @truncate_kernarg_i32_to_i16(ptr addrspace(1) %out, i32 %arg) nounwind {
%trunc = trunc i32 %arg to i16
@@ -32,7 +32,7 @@ define amdgpu_kernel void @truncate_buffer_load_i32_to_i16(ptr addrspace(1) %out
}
; FUNC-LABEL: {{^}}truncate_kernarg_i32_to_i8:
-; SI: s_load_dword s
+; SI: s_load_dwordx4
; SI: buffer_store_byte v
define amdgpu_kernel void @truncate_kernarg_i32_to_i8(ptr addrspace(1) %out, i32 %arg) nounwind {
%trunc = trunc i32 %arg to i8
@@ -54,7 +54,7 @@ define amdgpu_kernel void @truncate_buffer_load_i32_to_i8(ptr addrspace(1) %out,
}
; FUNC-LABEL: {{^}}truncate_kernarg_i32_to_i1:
-; SI: s_load_dword s
+; SI: s_load_dwordx4
; SI: buffer_store_byte v
define amdgpu_kernel void @truncate_kernarg_i32_to_i1(ptr addrspace(1) %out, i32 %arg) nounwind {
%trunc = trunc i32 %arg to i1
@@ -76,7 +76,8 @@ define amdgpu_kernel void @truncate_buffer_load_i32_to_i1(ptr addrspace(1) %out,
}
; FUNC-LABEL: {{^}}truncate_kernarg_i64_to_i32:
-; SI: s_load_dword s
+; SI-DAG: s_load_dword s{{[0-9]+}},
+; SI-DAG: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}
; SI: buffer_store_dword v
define amdgpu_kernel void @truncate_kernarg_i64_to_i32(ptr addrspace(1) %out, [8 x i32], i64 %arg) nounwind {
%trunc = trunc i64 %arg to i32
@@ -98,7 +99,8 @@ define amdgpu_kernel void @truncate_buffer_load_i64_to_i32(ptr addrspace(1) %out
}
; FUNC-LABEL: {{^}}srl_kernarg_i64_to_i32:
-; SI: s_load_dword s
+; SI-DAG: s_load_dword s{{[0-9]+}},
+; SI-DAG: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}
; SI: buffer_store_dword v
define amdgpu_kernel void @srl_kernarg_i64_to_i32(ptr addrspace(1) %out, [8 x i32], i64 %arg) nounwind {
%srl = lshr i64 %arg, 32
@@ -123,7 +125,7 @@ define amdgpu_kernel void @srl_buffer_load_i64_to_i32(ptr addrspace(1) %out, ptr
; Might as well reduce to 8-bit loads.
; FUNC-LABEL: {{^}}truncate_kernarg_i16_to_i8:
-; SI: s_load_dword s
+; SI: s_load_dwordx4
; SI: buffer_store_byte v
define amdgpu_kernel void @truncate_kernarg_i16_to_i8(ptr addrspace(1) %out, i16 %arg) nounwind {
%trunc = trunc i16 %arg to i8
@@ -145,7 +147,8 @@ define amdgpu_kernel void @truncate_buffer_load_i16_to_i8(ptr addrspace(1) %out,
}
; FUNC-LABEL: {{^}}srl_kernarg_i64_to_i8:
-; SI: s_load_dword s
+; SI-DAG: s_load_dword s{{[0-9]+}},
+; SI-DAG: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}
; SI: buffer_store_byte v
define amdgpu_kernel void @srl_kernarg_i64_to_i8(ptr addrspace(1) %out, [8 x i32], i64 %arg) nounwind {
%srl = lshr i64 %arg, 32
@@ -169,7 +172,8 @@ define amdgpu_kernel void @srl_buffer_load_i64_to_i8(ptr addrspace(1) %out, ptr
}
; FUNC-LABEL: {{^}}truncate_kernarg_i64_to_i8:
-; SI: s_load_dword s
+; SI-DAG: s_load_dword s{{[0-9]+}},
+; SI-DAG: s_load_dwordx2 s{{\[[0-9]+:[0-9]+\]}}
; SI: buffer_store_byte v
define amdgpu_kernel void @truncate_kernarg_i64_to_i8(ptr addrspace(1) %out, [8 x i32], i64 %arg) nounwind {
%trunc = trunc i64 %arg to i8
@@ -191,7 +195,9 @@ define amdgpu_kernel void @truncate_buffer_load_i64_to_i8(ptr addrspace(1) %out,
}
; FUNC-LABEL: {{^}}smrd_mask_i32_to_i16
-; SI: s_load_dword [[LOAD:s[0-9]+]], s[{{[0-9]+}}:{{[0-9]+}}], 0x0
+; SI: s_load_dwordx4
+; SI: s_waitcnt lgkmcnt(0)
+; SI: s_load_dword [[LOAD:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x0
; SI: s_waitcnt lgkmcnt(0)
; SI: s_and_b32 s{{[0-9]+}}, [[LOAD]], 0xffff
define amdgpu_kernel void @smrd_mask_i32_to_i16(ptr addrspace(1) %out, ptr addrspace(4) %in) {
diff --git a/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll b/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
index 5155873afa8f6..c8f65dd0e5291 100644
--- a/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
+++ b/llvm/test/CodeGen/AMDGPU/noclobber-barrier.ll
@@ -72,8 +72,7 @@ define amdgpu_kernel void @memory_phi_no_clobber(ptr addrspace(1) %arg, i1 %cond
;
; GCN-LABEL: memory_phi_no_clobber:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp0_b32 s2, 0
; GCN-NEXT: s_load_dword s4, s[0:1], 0x0
@@ -139,8 +138,7 @@ define amdgpu_kernel void @memory_phi_clobber1(ptr addrspace(1) %arg, i1 %cond)
;
; GCN-LABEL: memory_phi_clobber1:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp0_b32 s2, 0
; GCN-NEXT: s_load_dword s4, s[0:1], 0x0
@@ -208,8 +206,7 @@ define amdgpu_kernel void @memory_phi_clobber2(ptr addrspace(1) %arg, i1 %cond)
;
; GCN-LABEL: memory_phi_clobber2:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_bitcmp0_b32 s2, 0
; GCN-NEXT: s_mov_b64 s[2:3], -1
@@ -273,24 +270,23 @@ define amdgpu_kernel void @no_clobbering_loop1(ptr addrspace(1) %arg, i1 %cc) {
;
; GCN-LABEL: no_clobbering_loop1:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GCN-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GCN-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GCN-NEXT: v_mov_b32_e32 v0, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_bitcmp1_b32 s0, 0
-; GCN-NEXT: s_load_dword s4, s[2:3], 0x0
+; GCN-NEXT: s_bitcmp1_b32 s10, 0
+; GCN-NEXT: s_load_dword s2, s[8:9], 0x0
; GCN-NEXT: s_cselect_b64 s[0:1], -1, 0
; GCN-NEXT: s_xor_b64 s[0:1], s[0:1], -1
; GCN-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]
; GCN-NEXT: v_cmp_ne_u32_e64 s[0:1], 1, v1
; GCN-NEXT: .LBB4_1: ; %while.cond
; GCN-NEXT: ; =>This Inner Loop Header: Depth=1
-; GCN-NEXT: s_load_dword s5, s[2:3], 0x4
+; GCN-NEXT: s_load_dword s3, s[8:9], 0x4
; GCN-NEXT: s_and_b64 vcc, exec, s[0:1]
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_add_i32 s5, s5, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: global_store_dword v0, v1, s[2:3] offset:8
+; GCN-NEXT: s_add_i32 s3, s3, s2
+; GCN-NEXT: v_mov_b32_e32 v1, s3
+; GCN-NEXT: global_store_dword v0, v1, s[8:9] offset:8
; GCN-NEXT: ; wave barrier
; GCN-NEXT: s_cbranch_vccnz .LBB4_1
; GCN-NEXT: ; %bb.2: ; %end
diff --git a/llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll b/llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll
index 70095e4e5932a..181c473aea9be 100644
--- a/llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll
+++ b/llvm/test/CodeGen/AMDGPU/non-entry-alloca.ll
@@ -18,31 +18,31 @@ define amdgpu_kernel void @kernel_non_entry_block_static_alloca_uniformly_reache
; MUBUF-LABEL: kernel_non_entry_block_static_alloca_uniformly_reached_align4:
; MUBUF: ; %bb.0: ; %entry
; MUBUF-NEXT: s_add_u32 s0, s0, s9
-; MUBUF-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x8
+; MUBUF-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x0
; MUBUF-NEXT: s_addc_u32 s1, s1, 0
; MUBUF-NEXT: s_mov_b32 s33, 0
; MUBUF-NEXT: s_movk_i32 s32, 0x400
; MUBUF-NEXT: s_waitcnt lgkmcnt(0)
-; MUBUF-NEXT: s_cmp_lg_u32 s8, 0
+; MUBUF-NEXT: s_cmp_lg_u32 s10, 0
; MUBUF-NEXT: s_cbranch_scc1 .LBB0_3
; MUBUF-NEXT: ; %bb.1: ; %bb.0
-; MUBUF-NEXT: s_cmp_lg_u32 s9, 0
+; MUBUF-NEXT: s_load_dword s4, s[4:5], 0x10
+; MUBUF-NEXT: s_cmp_lg_u32 s11, 0
; MUBUF-NEXT: s_cbranch_scc1 .LBB0_3
; MUBUF-NEXT: ; %bb.2: ; %bb.1
-; MUBUF-NEXT: s_mov_b32 s6, s32
+; MUBUF-NEXT: s_mov_b32 s5, s32
; MUBUF-NEXT: v_mov_b32_e32 v1, 0
; MUBUF-NEXT: v_mov_b32_e32 v2, 1
-; MUBUF-NEXT: s_add_i32 s32, s6, 0x1000
-; MUBUF-NEXT: buffer_store_dword v1, off, s[0:3], s6
-; MUBUF-NEXT: buffer_store_dword v2, off, s[0:3], s6 offset:4
-; MUBUF-NEXT: s_lshl2_add_u32 s6, s10, s6
-; MUBUF-NEXT: v_mov_b32_e32 v2, s6
+; MUBUF-NEXT: s_waitcnt lgkmcnt(0)
+; MUBUF-NEXT: s_lshl2_add_u32 s4, s4, s5
+; MUBUF-NEXT: buffer_store_dword v1, off, s[0:3], s5
+; MUBUF-NEXT: buffer_store_dword v2, off, s[0:3], s5 offset:4
+; MUBUF-NEXT: v_mov_b32_e32 v2, s4
; MUBUF-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; MUBUF-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x0
+; MUBUF-NEXT: s_add_i32 s32, s5, 0x1000
; MUBUF-NEXT: s_waitcnt vmcnt(0)
; MUBUF-NEXT: v_add_u32_e32 v0, v2, v0
-; MUBUF-NEXT: s_waitcnt lgkmcnt(0)
-; MUBUF-NEXT: global_store_dword v1, v0, s[4:5]
+; MUBUF-NEXT: global_store_dword v1, v0, s[8:9]
; MUBUF-NEXT: .LBB0_3: ; %bb.2
; MUBUF-NEXT: v_mov_b32_e32 v0, 0
; MUBUF-NEXT: global_store_dword v[0:1], v0, off
@@ -52,29 +52,29 @@ define amdgpu_kernel void @kernel_non_entry_block_static_alloca_uniformly_reache
; FLATSCR-LABEL: kernel_non_entry_block_static_alloca_uniformly_reached_align4:
; FLATSCR: ; %bb.0: ; %entry
; FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s2, s5
-; FLATSCR-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x8
+; FLATSCR-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x0
; FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s3, 0
; FLATSCR-NEXT: s_mov_b32 s33, 0
; FLATSCR-NEXT: s_mov_b32 s32, 16
; FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; FLATSCR-NEXT: s_cmp_lg_u32 s4, 0
+; FLATSCR-NEXT: s_cmp_lg_u32 s6, 0
; FLATSCR-NEXT: s_cbranch_scc1 .LBB0_3
; FLATSCR-NEXT: ; %bb.1: ; %bb.0
-; FLATSCR-NEXT: s_cmp_lg_u32 s5, 0
+; FLATSCR-NEXT: s_load_dword s0, s[0:1], 0x10
+; FLATSCR-NEXT: s_cmp_lg_u32 s7, 0
; FLATSCR-NEXT: s_cbranch_scc1 .LBB0_3
; FLATSCR-NEXT: ; %bb.2: ; %bb.1
-; FLATSCR-NEXT: s_mov_b32 s2, s32
+; FLATSCR-NEXT: s_mov_b32 s1, s32
; FLATSCR-NEXT: v_mov_b32_e32 v1, 0
; FLATSCR-NEXT: v_mov_b32_e32 v2, 1
-; FLATSCR-NEXT: s_add_i32 s32, s2, 64
-; FLATSCR-NEXT: scratch_store_dwordx2 off, v[1:2], s2
-; FLATSCR-NEXT: s_lshl2_add_u32 s2, s6, s2
-; FLATSCR-NEXT: scratch_load_dword v2, off, s2
-; FLATSCR-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
+; FLATSCR-NEXT: scratch_store_dwordx2 off, v[1:2], s1
+; FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
+; FLATSCR-NEXT: s_lshl2_add_u32 s0, s0, s1
+; FLATSCR-NEXT: scratch_load_dword v2, off, s0
+; FLATSCR-NEXT: s_add_i32 s32, s1, 64
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
; FLATSCR-NEXT: v_add_u32_e32 v0, v2, v0
-; FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; FLATSCR-NEXT: global_store_dword v1, v0, s[0:1]
+; FLATSCR-NEXT: global_store_dword v1, v0, s[4:5]
; FLATSCR-NEXT: .LBB0_3: ; %bb.2
; FLATSCR-NEXT: v_mov_b32_e32 v0, 0
; FLATSCR-NEXT: global_store_dword v[0:1], v0, off
@@ -118,30 +118,28 @@ bb.2:
define amdgpu_kernel void @kernel_non_entry_block_static_alloca_uniformly_reached_align64(ptr addrspace(1) %out, i32 %arg.cond, i32 %in) #2 {
; MUBUF-LABEL: kernel_non_entry_block_static_alloca_uniformly_reached_align64:
; MUBUF: ; %bb.0: ; %entry
-; MUBUF-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
+; MUBUF-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
; MUBUF-NEXT: s_add_u32 s0, s0, s17
; MUBUF-NEXT: s_addc_u32 s1, s1, 0
; MUBUF-NEXT: s_mov_b32 s33, 0
; MUBUF-NEXT: s_movk_i32 s32, 0x1000
; MUBUF-NEXT: s_waitcnt lgkmcnt(0)
-; MUBUF-NEXT: s_cmp_lg_u32 s4, 0
+; MUBUF-NEXT: s_cmp_lg_u32 s6, 0
; MUBUF-NEXT: s_cbranch_scc1 .LBB1_2
; MUBUF-NEXT: ; %bb.1: ; %bb.0
-; MUBUF-NEXT: s_add_i32 s4, s32, 0xfff
-; MUBUF-NEXT: s_and_b32 s4, s4, 0xfffff000
-; MUBUF-NEXT: s_add_i32 s32, s4, 0x1000
+; MUBUF-NEXT: s_add_i32 s6, s32, 0xfff
+; MUBUF-NEXT: s_and_b32 s6, s6, 0xfffff000
+; MUBUF-NEXT: s_add_i32 s32, s6, 0x1000
; MUBUF-NEXT: v_mov_b32_e32 v1, 0
-; MUBUF-NEXT: v_mov_b32_e32 v2, s4
+; MUBUF-NEXT: v_mov_b32_e32 v2, s6
; MUBUF-NEXT: v_mov_b32_e32 v3, 1
-; MUBUF-NEXT: s_lshl2_add_u32 s4, s5, s4
+; MUBUF-NEXT: s_lshl2_add_u32 s6, s7, s6
; MUBUF-NEXT: buffer_store_dword v1, v2, s[0:3], 0 offen
; MUBUF-NEXT: buffer_store_dword v3, v2, s[0:3], 0 offen offset:4
-; MUBUF-NEXT: v_mov_b32_e32 v2, s4
+; MUBUF-NEXT: v_mov_b32_e32 v2, s6
; MUBUF-NEXT: buffer_load_dword v2, v2, s[0:3], 0 offen
-; MUBUF-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; MUBUF-NEXT: s_waitcnt vmcnt(0)
; MUBUF-NEXT: v_add_u32_e32 v0, v2, v0
-; MUBUF-NEXT: s_waitcnt lgkmcnt(0)
; MUBUF-NEXT: global_store_dword v1, v0, s[4:5]
; MUBUF-NEXT: .LBB1_2: ; %bb.1
; MUBUF-NEXT: v_mov_b32_e32 v0, 0
@@ -151,27 +149,25 @@ define amdgpu_kernel void @kernel_non_entry_block_static_alloca_uniformly_reache
;
; FLATSCR-LABEL: kernel_non_entry_block_static_alloca_uniformly_reached_align64:
; FLATSCR: ; %bb.0: ; %entry
-; FLATSCR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
+; FLATSCR-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; FLATSCR-NEXT: s_add_u32 flat_scratch_lo, s8, s13
; FLATSCR-NEXT: s_addc_u32 flat_scratch_hi, s9, 0
; FLATSCR-NEXT: s_mov_b32 s33, 0
; FLATSCR-NEXT: s_mov_b32 s32, 64
; FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; FLATSCR-NEXT: s_cmp_lg_u32 s0, 0
+; FLATSCR-NEXT: s_cmp_lg_u32 s2, 0
; FLATSCR-NEXT: s_cbranch_scc1 .LBB1_2
; FLATSCR-NEXT: ; %bb.1: ; %bb.0
-; FLATSCR-NEXT: s_add_i32 s0, s32, 63
+; FLATSCR-NEXT: s_add_i32 s2, s32, 63
; FLATSCR-NEXT: v_mov_b32_e32 v1, 0
-; FLATSCR-NEXT: s_andn2_b32 s0, s0, 63
+; FLATSCR-NEXT: s_andn2_b32 s2, s2, 63
; FLATSCR-NEXT: v_mov_b32_e32 v2, 1
-; FLATSCR-NEXT: s_add_i32 s32, s0, 64
-; FLATSCR-NEXT: scratch_store_dwordx2 off, v[1:2], s0
-; FLATSCR-NEXT: s_lshl2_add_u32 s0, s1, s0
-; FLATSCR-NEXT: scratch_load_dword v2, off, s0
-; FLATSCR-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; FLATSCR-NEXT: s_add_i32 s32, s2, 64
+; FLATSCR-NEXT: scratch_store_dwordx2 off, v[1:2], s2
+; FLATSCR-NEXT: s_lshl2_add_u32 s2, s3, s2
+; FLATSCR-NEXT: scratch_load_dword v2, off, s2
; FLATSCR-NEXT: s_waitcnt vmcnt(0)
; FLATSCR-NEXT: v_add_u32_e32 v0, v2, v0
-; FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
; FLATSCR-NEXT: global_store_dword v1, v0, s[0:1]
; FLATSCR-NEXT: .LBB1_2: ; %bb.1
; FLATSCR-NEXT: v_mov_b32_e32 v0, 0
diff --git a/llvm/test/CodeGen/AMDGPU/or.ll b/llvm/test/CodeGen/AMDGPU/or.ll
index 9afaab5ebcfb6..86a76689b18aa 100644
--- a/llvm/test/CodeGen/AMDGPU/or.ll
+++ b/llvm/test/CodeGen/AMDGPU/or.ll
@@ -136,25 +136,27 @@ define amdgpu_kernel void @scalar_or_i32(ptr addrspace(1) %out, i32 %a, i32 %b)
; GFX6-LABEL: scalar_or_i32:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_or_b32 s4, s4, s5
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_or_b32 s0, s2, s3
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: scalar_or_i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX8-NEXT: s_or_b32 s4, s4, s5
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_or_b32 s0, s2, s3
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
%or = or i32 %a, %b
store i32 %or, ptr addrspace(1) %out
@@ -208,26 +210,28 @@ define amdgpu_kernel void @vector_or_i32(ptr addrspace(1) %out, ptr addrspace(1)
define amdgpu_kernel void @scalar_or_literal_i32(ptr addrspace(1) %out, i32 %a) {
; GFX6-LABEL: scalar_or_literal_i32:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_or_b32 s4, s6, 0x1869f
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_or_b32 s0, s2, 0x1869f
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: scalar_or_literal_i32:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_or_b32 s4, s6, 0x1869f
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_or_b32 s0, s2, 0x1869f
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
%or = or i32 %a, 99999
store i32 %or, ptr addrspace(1) %out, align 4
@@ -924,14 +928,16 @@ define amdgpu_kernel void @or_i1(ptr addrspace(1) %out, ptr addrspace(1) %in0, p
define amdgpu_kernel void @s_or_i1(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c, i32 %d) {
; GFX6-LABEL: s_or_i1:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; GFX6-NEXT: s_mov_b32 s7, 0xf000
; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_cmp_eq_u32 s0, s1
-; GFX6-NEXT: s_cselect_b64 s[0:1], -1, 0
; GFX6-NEXT: s_cmp_eq_u32 s2, s3
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX6-NEXT: s_cmp_eq_u32 s8, s9
; GFX6-NEXT: s_cselect_b64 s[2:3], -1, 0
; GFX6-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX6-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
@@ -940,14 +946,16 @@ define amdgpu_kernel void @s_or_i1(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c
;
; GFX8-LABEL: s_or_i1:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
; GFX8-NEXT: s_mov_b32 s7, 0xf000
; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_cmp_eq_u32 s0, s1
-; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
; GFX8-NEXT: s_cmp_eq_u32 s2, s3
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: s_cselect_b64 s[0:1], -1, 0
+; GFX8-NEXT: s_cmp_eq_u32 s8, s9
; GFX8-NEXT: s_cselect_b64 s[2:3], -1, 0
; GFX8-NEXT: s_or_b64 s[0:1], s[0:1], s[2:3]
; GFX8-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
index 27ea6f344796f..d9ab389e7285c 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
@@ -76,19 +76,33 @@ define amdgpu_kernel void @fadd_v2_vs(ptr addrspace(1) %a, <2 x float> %x) {
; PACKED-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; PACKED-NEXT: s_endpgm
;
-; GFX1250-LABEL: fadd_v2_vs:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
-; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
-; GFX1250-NEXT: global_store_b64 v4, v[0:1], s[0:1] scale_offset
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: fadd_v2_vs:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: fadd_v2_vs:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT: global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT: s_endpgm
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
%load = load <2 x float>, ptr addrspace(1) %gep, align 8
@@ -904,8 +918,7 @@ define amdgpu_kernel void @fadd_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, float %x) {
; GFX900-LABEL: fadd_v2_v_fneg:
; GFX900: ; %bb.0:
-; GFX900-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX900-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
@@ -917,8 +930,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, float %x) {
;
; PACKED-SDAG-LABEL: fadd_v2_v_fneg:
; PACKED-SDAG: ; %bb.0:
-; PACKED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; PACKED-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; PACKED-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; PACKED-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; PACKED-SDAG-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -983,8 +995,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg(ptr addrspace(1) %a, float %x) {
define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, float %x) {
; GFX900-LABEL: fadd_v2_v_fneg_lo:
; GFX900: ; %bb.0:
-; GFX900-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX900-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
@@ -996,8 +1007,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, float %x) {
;
; PACKED-SDAG-LABEL: fadd_v2_v_fneg_lo:
; PACKED-SDAG: ; %bb.0:
-; PACKED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; PACKED-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; PACKED-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; PACKED-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; PACKED-SDAG-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -1062,8 +1072,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo(ptr addrspace(1) %a, float %x) {
define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, float %x) {
; GFX900-LABEL: fadd_v2_v_fneg_hi:
; GFX900: ; %bb.0:
-; GFX900-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX900-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
@@ -1075,8 +1084,7 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi(ptr addrspace(1) %a, float %x) {
;
; PACKED-SDAG-LABEL: fadd_v2_v_fneg_hi:
; PACKED-SDAG: ; %bb.0:
-; PACKED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; PACKED-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; PACKED-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; PACKED-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; PACKED-SDAG-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -1158,8 +1166,9 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo2(ptr addrspace(1) %a, float %x, flo
; PACKED-SDAG-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-SDAG-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
+; PACKED-SDAG-NEXT: s_xor_b32 s2, s2, 0x80000000
; PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
-; PACKED-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3] neg_lo:[0,1]
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3]
; PACKED-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; PACKED-SDAG-NEXT: s_endpgm
;
@@ -1184,10 +1193,11 @@ define amdgpu_kernel void @fadd_v2_v_fneg_lo2(ptr addrspace(1) %a, float %x, flo
; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
-; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-SDAG-NEXT: s_xor_b32 s2, s2, 0x80000000
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v3, s3 :: v_dual_mov_b32 v2, s2
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3] neg_lo:[0,1]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: global_store_b64 v4, v[0:1], s[0:1] scale_offset
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -1236,8 +1246,10 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi2(ptr addrspace(1) %a, float %x, flo
; PACKED-SDAG-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-SDAG-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
+; PACKED-SDAG-NEXT: s_xor_b32 s5, s2, 0x80000000
+; PACKED-SDAG-NEXT: s_mov_b32 s4, s3
; PACKED-SDAG-NEXT: s_waitcnt vmcnt(0)
-; PACKED-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[2:3] op_sel:[0,1] op_sel_hi:[1,0] neg_hi:[0,1]
+; PACKED-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], s[4:5]
; PACKED-SDAG-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; PACKED-SDAG-NEXT: s_endpgm
;
@@ -1262,10 +1274,11 @@ define amdgpu_kernel void @fadd_v2_v_fneg_hi2(ptr addrspace(1) %a, float %x, flo
; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
-; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-SDAG-NEXT: s_xor_b32 s2, s2, 0x80000000
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, s2
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3] op_sel:[0,1] op_sel_hi:[1,0] neg_hi:[0,1]
+; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3]
; GFX1250-SDAG-NEXT: global_store_b64 v4, v[0:1], s[0:1] scale_offset
; GFX1250-SDAG-NEXT: s_endpgm
;
@@ -1363,19 +1376,33 @@ define amdgpu_kernel void @fmul_v2_vs(ptr addrspace(1) %a, <2 x float> %x) {
; PACKED-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; PACKED-NEXT: s_endpgm
;
-; GFX1250-LABEL: fmul_v2_vs:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
-; GFX1250-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[2:3]
-; GFX1250-NEXT: global_store_b64 v4, v[0:1], s[0:1] scale_offset
-; GFX1250-NEXT: s_endpgm
+; GFX1250-SDAG-LABEL: fmul_v2_vs:
+; GFX1250-SDAG: ; %bb.0:
+; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x0
+; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-SDAG-NEXT: global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-SDAG-NEXT: s_endpgm
+;
+; GFX1250-GISEL-LABEL: fmul_v2_vs:
+; GFX1250-GISEL: ; %bb.0:
+; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-GISEL-NEXT: v_and_b32_e32 v4, 0x3ff, v0
+; GFX1250-GISEL-NEXT: s_wait_kmcnt 0x0
+; GFX1250-GISEL-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset
+; GFX1250-GISEL-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
+; GFX1250-GISEL-NEXT: s_wait_loadcnt 0x0
+; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[2:3]
+; GFX1250-GISEL-NEXT: global_store_b64 v4, v[0:1], s[0:1] scale_offset
+; GFX1250-GISEL-NEXT: s_endpgm
%id = tail call i32 @llvm.amdgcn.workitem.id.x()
%gep = getelementptr inbounds <2 x float>, ptr addrspace(1) %a, i32 %id
%load = load <2 x float>, ptr addrspace(1) %gep, align 8
@@ -2066,8 +2093,7 @@ define amdgpu_kernel void @fmul_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
define amdgpu_kernel void @fmul_v2_v_fneg(ptr addrspace(1) %a, float %x) {
; GFX900-LABEL: fmul_v2_v_fneg:
; GFX900: ; %bb.0:
-; GFX900-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX900-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
@@ -2079,8 +2105,7 @@ define amdgpu_kernel void @fmul_v2_v_fneg(ptr addrspace(1) %a, float %x) {
;
; PACKED-SDAG-LABEL: fmul_v2_v_fneg:
; PACKED-SDAG: ; %bb.0:
-; PACKED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; PACKED-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; PACKED-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; PACKED-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; PACKED-SDAG-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -3005,8 +3030,7 @@ define amdgpu_kernel void @fma_v2_v_unfoldable_lit(ptr addrspace(1) %a) {
define amdgpu_kernel void @fma_v2_v_fneg(ptr addrspace(1) %a, float %x) {
; GFX900-LABEL: fma_v2_v_fneg:
; GFX900: ; %bb.0:
-; GFX900-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX900-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GFX900-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: global_load_dwordx2 v[0:1], v2, s[0:1]
@@ -3018,8 +3042,7 @@ define amdgpu_kernel void @fma_v2_v_fneg(ptr addrspace(1) %a, float %x) {
;
; PACKED-SDAG-LABEL: fma_v2_v_fneg:
; PACKED-SDAG: ; %bb.0:
-; PACKED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; PACKED-SDAG-NEXT: s_load_dword s2, s[4:5], 0x2c
+; PACKED-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; PACKED-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; PACKED-SDAG-NEXT: v_lshlrev_b32_e32 v2, 3, v0
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -3277,13 +3300,12 @@ bb:
define amdgpu_kernel void @shuffle_add_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; GFX900-LABEL: shuffle_add_f32:
; GFX900: ; %bb.0: ; %bb
-; GFX900-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX900-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-NEXT: v_mov_b32_e32 v4, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s0
+; GFX900-NEXT: v_mov_b32_e32 v2, s2
; GFX900-NEXT: ds_read_b64 v[0:1], v2
; GFX900-NEXT: ds_read_b64 v[2:3], v2 offset:8
-; GFX900-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: v_add_f32_e32 v1, v1, v2
; GFX900-NEXT: v_add_f32_e32 v0, v0, v3
@@ -3292,13 +3314,12 @@ define amdgpu_kernel void @shuffle_add_f32(ptr addrspace(1) %out, ptr addrspace(
;
; PACKED-SDAG-LABEL: shuffle_add_f32:
; PACKED-SDAG: ; %bb.0: ; %bb
-; PACKED-SDAG-NEXT: s_load_dword s0, s[4:5], 0x2c
+; PACKED-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; PACKED-SDAG-NEXT: v_mov_b32_e32 v4, 0
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; PACKED-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; PACKED-SDAG-NEXT: v_mov_b32_e32 v2, s2
; PACKED-SDAG-NEXT: ds_read_b64 v[0:1], v2
; PACKED-SDAG-NEXT: ds_read_b64 v[2:3], v2 offset:8
-; PACKED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[2:3] op_sel:[0,1] op_sel_hi:[1,0]
; PACKED-SDAG-NEXT: global_store_dwordx2 v4, v[0:1], s[0:1]
@@ -3357,12 +3378,11 @@ bb:
define amdgpu_kernel void @shuffle_neg_add_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; GFX900-LABEL: shuffle_neg_add_f32:
; GFX900: ; %bb.0: ; %bb
-; GFX900-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GFX900-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX900-NEXT: v_mov_b32_e32 v4, 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: v_mov_b32_e32 v2, s0
+; GFX900-NEXT: v_mov_b32_e32 v2, s2
; GFX900-NEXT: ds_read_b64 v[0:1], v2
-; GFX900-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
; GFX900-NEXT: ds_read_b32 v3, v0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
@@ -3375,12 +3395,11 @@ define amdgpu_kernel void @shuffle_neg_add_f32(ptr addrspace(1) %out, ptr addrsp
;
; PACKED-SDAG-LABEL: shuffle_neg_add_f32:
; PACKED-SDAG: ; %bb.0: ; %bb
-; PACKED-SDAG-NEXT: s_load_dword s0, s[4:5], 0x2c
+; PACKED-SDAG-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; PACKED-SDAG-NEXT: v_mov_b32_e32 v4, 0
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
-; PACKED-SDAG-NEXT: v_mov_b32_e32 v2, s0
+; PACKED-SDAG-NEXT: v_mov_b32_e32 v2, s2
; PACKED-SDAG-NEXT: ds_read_b64 v[0:1], v2
-; PACKED-SDAG-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; PACKED-SDAG-NEXT: ds_read_b32 v3, v0
; PACKED-SDAG-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/permlane-op-sel.ll b/llvm/test/CodeGen/AMDGPU/permlane-op-sel.ll
index d21674d1b3a62..5ca906223c42e 100644
--- a/llvm/test/CodeGen/AMDGPU/permlane-op-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/permlane-op-sel.ll
@@ -4,10 +4,10 @@
declare i32 @llvm.amdgcn.permlane16(i32, i32, i32, i32, i1, i1)
; OBJ-LABEL: <permlane_op_sel>:
-; OBJ: v_permlane16_b32 v0, v0, s1, s2 op_sel:[1,0]
+; OBJ: v_permlane16_b32 v0, v0, s3, s4 op_sel:[1,0]
; ASM-LABEL: permlane_op_sel:
-; ASM: v_permlane16_b32 v0, v0, s1, s2 op_sel:[1,0] ; encoding: [0x00,0x08,0x77,0xd7,0x00,0x03,0x08,0x00]
+; ASM: v_permlane16_b32 v0, v0, s3, s4 op_sel:[1,0] ; encoding: [0x00,0x08,0x77,0xd7,0x00,0x07,0x10,0x00]
define amdgpu_kernel void @permlane_op_sel(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) {
%v = call i32 @llvm.amdgcn.permlane16(i32 %src0, i32 %src0, i32 %src1, i32 %src2, i1 1, i1 0)
store i32 %v, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/permute.ll b/llvm/test/CodeGen/AMDGPU/permute.ll
index 533f0672204ed..da1bc95c487e0 100644
--- a/llvm/test/CodeGen/AMDGPU/permute.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute.ll
@@ -4,8 +4,7 @@
define amdgpu_kernel void @lsh8_or_and(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: lsh8_or_and:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0x6050400
; GCN-NEXT: s_waitcnt lgkmcnt(0)
@@ -31,8 +30,7 @@ bb:
define amdgpu_kernel void @lsr24_or_and(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: lsr24_or_and:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0x7060503
; GCN-NEXT: s_waitcnt lgkmcnt(0)
@@ -58,8 +56,7 @@ bb:
define amdgpu_kernel void @and_or_lsr24(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: and_or_lsr24:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0x7060503
; GCN-NEXT: s_waitcnt lgkmcnt(0)
@@ -87,8 +84,7 @@ bb:
define amdgpu_kernel void @and_or_and(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: and_or_and:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0x7020500
; GCN-NEXT: s_waitcnt lgkmcnt(0)
@@ -115,8 +111,7 @@ bb:
define amdgpu_kernel void @lsh8_or_lsr24(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: lsh8_or_lsr24:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v1, s1
@@ -141,8 +136,7 @@ bb:
define amdgpu_kernel void @lsh16_or_lsr24(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: lsh16_or_lsr24:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0x5040c03
; GCN-NEXT: s_waitcnt lgkmcnt(0)
@@ -168,8 +162,7 @@ bb:
define amdgpu_kernel void @and_xor_and(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: and_xor_and:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0x7020104
; GCN-NEXT: s_waitcnt lgkmcnt(0)
@@ -196,8 +189,7 @@ bb:
define amdgpu_kernel void @and_or_or_and(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: and_or_or_and:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v1, s1
@@ -226,8 +218,7 @@ bb:
define amdgpu_kernel void @and_or_and_shl(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: and_or_and_shl:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0x50c0c00
; GCN-NEXT: s_waitcnt lgkmcnt(0)
@@ -254,8 +245,7 @@ bb:
define amdgpu_kernel void @or_and_or(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: or_and_or:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0x7020104
; GCN-NEXT: s_waitcnt lgkmcnt(0)
@@ -282,8 +272,7 @@ bb:
define amdgpu_kernel void @known_ffff0500(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: known_ffff0500:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v5, 0xffff8004
; GCN-NEXT: s_waitcnt lgkmcnt(0)
@@ -322,8 +311,7 @@ bb:
define amdgpu_kernel void @known_050c0c00(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: known_050c0c00:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v5, 0x50c0c00
; GCN-NEXT: v_mov_b32_e32 v6, 4
@@ -358,8 +346,7 @@ bb:
define amdgpu_kernel void @known_ffff8004(ptr addrspace(1) nocapture %arg, i32 %arg1) {
; GCN-LABEL: known_ffff8004:
; GCN: ; %bb.0: ; %bb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GCN-NEXT: s_load_dword s2, s[4:5], 0x2c
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GCN-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GCN-NEXT: v_mov_b32_e32 v5, 0xffff0500
; GCN-NEXT: v_mov_b32_e32 v6, 0xffff8004
diff --git a/llvm/test/CodeGen/AMDGPU/permute_i8.ll b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
index b54d80a03abd0..a3322a1d54177 100644
--- a/llvm/test/CodeGen/AMDGPU/permute_i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/permute_i8.ll
@@ -3902,19 +3902,19 @@ define amdgpu_kernel void @more_any_extend_to_perm(i8 %arg, <4 x i8> %arg1, i8 %
; GFX10-LABEL: more_any_extend_to_perm:
; GFX10: ; %bb.0: ; %bb
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dword s6, s[4:5], 0x34
; GFX10-NEXT: v_mov_b32_e32 v0, 0xc0c0304
; GFX10-NEXT: v_mov_b32_e32 v1, 0xc0c0104
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_perm_b32 v2, s6, s7, v0
-; GFX10-NEXT: v_perm_b32 v0, s0, s7, v0
-; GFX10-NEXT: v_perm_b32 v3, s0, s7, v1
-; GFX10-NEXT: v_perm_b32 v1, s6, s7, v1
-; GFX10-NEXT: v_mov_b32_e32 v4, s1
+; GFX10-NEXT: v_perm_b32 v2, s0, s1, v0
+; GFX10-NEXT: v_perm_b32 v0, s2, s1, v0
+; GFX10-NEXT: v_perm_b32 v3, s2, s1, v1
+; GFX10-NEXT: v_perm_b32 v1, s0, s1, v1
+; GFX10-NEXT: v_mov_b32_e32 v4, s3
; GFX10-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX10-NEXT: v_mov_b32_e32 v5, s2
+; GFX10-NEXT: v_mov_b32_e32 v5, s6
; GFX10-NEXT: v_or_b32_e32 v2, v3, v2
; GFX10-NEXT: v_or_b32_e32 v0, v1, v0
; GFX10-NEXT: ds_write_b32 v4, v2
@@ -3923,22 +3923,22 @@ define amdgpu_kernel void @more_any_extend_to_perm(i8 %arg, <4 x i8> %arg1, i8 %
;
; GFX9-LABEL: more_any_extend_to_perm:
; GFX9: ; %bb.0: ; %bb
-; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
-; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0xc0c0304
+; GFX9-NEXT: s_load_dword s4, s[4:5], 0x34
; GFX9-NEXT: v_mov_b32_e32 v0, 0xc0c0104
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v2, s7
-; GFX9-NEXT: v_perm_b32 v4, s6, v2, v1
-; GFX9-NEXT: v_perm_b32 v1, s0, v2, v1
-; GFX9-NEXT: v_perm_b32 v3, s0, v2, v0
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
+; GFX9-NEXT: v_perm_b32 v4, s0, v2, v1
+; GFX9-NEXT: v_perm_b32 v1, s2, v2, v1
+; GFX9-NEXT: v_perm_b32 v3, s2, v2, v0
; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX9-NEXT: v_perm_b32 v0, s6, v2, v0
+; GFX9-NEXT: v_perm_b32 v0, s0, v2, v0
; GFX9-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX9-NEXT: v_or_b32_e32 v3, v3, v4
-; GFX9-NEXT: v_mov_b32_e32 v4, s1
+; GFX9-NEXT: v_mov_b32_e32 v4, s3
; GFX9-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: v_mov_b32_e32 v1, s4
; GFX9-NEXT: ds_write_b32 v4, v3
; GFX9-NEXT: ds_write_b32 v1, v0
; GFX9-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/pr155452.ll b/llvm/test/CodeGen/AMDGPU/pr155452.ll
index b51b6ee07aea5..2a5e5ae2643d2 100644
--- a/llvm/test/CodeGen/AMDGPU/pr155452.ll
+++ b/llvm/test/CodeGen/AMDGPU/pr155452.ll
@@ -9,16 +9,15 @@ define amdgpu_kernel void @my_kernel(i64 %foo, i32 %bar) {
; CHECK-NEXT: s_mov_b32 flat_scratch_lo, s13
; CHECK-NEXT: s_add_i32 s12, s12, s17
; CHECK-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CHECK-NEXT: s_load_dword s0, s[8:9], 0x2
-; CHECK-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x0
-; CHECK-NEXT: s_mov_b64 s[4:5], 1
+; CHECK-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x0
+; CHECK-NEXT: s_mov_b64 s[2:3], 1
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: v_mov_b32_e32 v1, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_ashr_i32 s6, s0, 31
-; CHECK-NEXT: s_abs_i32 s7, s0
-; CHECK-NEXT: v_cvt_f32_u32_e32 v2, s7
-; CHECK-NEXT: s_sub_i32 s0, 0, s7
+; CHECK-NEXT: s_ashr_i32 s7, s6, 31
+; CHECK-NEXT: s_abs_i32 s6, s6
+; CHECK-NEXT: v_cvt_f32_u32_e32 v2, s6
+; CHECK-NEXT: s_sub_i32 s0, 0, s6
; CHECK-NEXT: v_rcp_f32_e32 v2, v2
; CHECK-NEXT: v_mul_f32_e32 v2, 0x4f7ffffe, v2
; CHECK-NEXT: v_cvt_u32_f32_e32 v2, v2
@@ -28,39 +27,39 @@ define amdgpu_kernel void @my_kernel(i64 %foo, i32 %bar) {
; CHECK-NEXT: s_and_b64 s[0:1], exec, -1
; CHECK-NEXT: .LBB0_1: ; %loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: v_mov_b32_e32 v3, s4
-; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[8:9], s2, v3, 1
-; CHECK-NEXT: s_mul_i32 s4, s3, s4
-; CHECK-NEXT: s_mul_i32 s5, s2, s5
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, s4, v4
-; CHECK-NEXT: v_readfirstlane_b32 s4, v3
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, s5, v4
-; CHECK-NEXT: s_ashr_i32 s5, s4, 31
-; CHECK-NEXT: s_abs_i32 s8, s4
-; CHECK-NEXT: s_xor_b32 s5, s5, s6
+; CHECK-NEXT: v_mov_b32_e32 v3, s2
+; CHECK-NEXT: v_mad_u64_u32 v[3:4], s[8:9], s4, v3, 1
+; CHECK-NEXT: s_mul_i32 s2, s5, s2
+; CHECK-NEXT: s_mul_i32 s3, s4, s3
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, s2, v4
+; CHECK-NEXT: v_readfirstlane_b32 s2, v3
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, s3, v4
+; CHECK-NEXT: s_ashr_i32 s3, s2, 31
+; CHECK-NEXT: s_abs_i32 s8, s2
+; CHECK-NEXT: s_xor_b32 s3, s3, s7
; CHECK-NEXT: v_mul_hi_u32 v3, s8, v2
; CHECK-NEXT: v_readfirstlane_b32 s9, v3
; CHECK-NEXT: v_add_i32_e32 v5, vcc, 1, v3
-; CHECK-NEXT: s_mul_i32 s9, s9, s7
+; CHECK-NEXT: s_mul_i32 s9, s9, s6
; CHECK-NEXT: s_sub_i32 s8, s8, s9
-; CHECK-NEXT: s_sub_i32 s9, s8, s7
-; CHECK-NEXT: s_cmp_ge_u32 s8, s7
+; CHECK-NEXT: s_sub_i32 s9, s8, s6
+; CHECK-NEXT: s_cmp_ge_u32 s8, s6
; CHECK-NEXT: s_cselect_b64 vcc, -1, 0
; CHECK-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
; CHECK-NEXT: s_cselect_b32 s8, s9, s8
; CHECK-NEXT: v_add_i32_e32 v5, vcc, 1, v3
-; CHECK-NEXT: s_cmp_ge_u32 s8, s7
+; CHECK-NEXT: s_cmp_ge_u32 s8, s6
; CHECK-NEXT: s_cselect_b64 vcc, -1, 0
; CHECK-NEXT: v_cndmask_b32_e32 v3, v3, v5, vcc
-; CHECK-NEXT: v_xor_b32_e32 v3, s5, v3
-; CHECK-NEXT: v_subrev_i32_e32 v3, vcc, s5, v3
+; CHECK-NEXT: v_xor_b32_e32 v3, s3, v3
+; CHECK-NEXT: v_subrev_i32_e32 v3, vcc, s3, v3
; CHECK-NEXT: v_ashrrev_i32_e32 v5, 31, v3
-; CHECK-NEXT: v_or_b32_e32 v3, s4, v3
+; CHECK-NEXT: v_or_b32_e32 v3, s2, v3
; CHECK-NEXT: v_or_b32_e32 v4, v4, v5
; CHECK-NEXT: flat_load_dwordx2 v[3:4], v[3:4]
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: flat_store_dwordx2 v[0:1], v[3:4]
-; CHECK-NEXT: s_mov_b64 s[4:5], 0
+; CHECK-NEXT: s_mov_b64 s[2:3], 0
; CHECK-NEXT: s_mov_b64 vcc, s[0:1]
; CHECK-NEXT: s_cbranch_vccnz .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %DummyReturnBlock
diff --git a/llvm/test/CodeGen/AMDGPU/private-memory-atomics.ll b/llvm/test/CodeGen/AMDGPU/private-memory-atomics.ll
index 9d3a84216c586..5f303f3f00d72 100644
--- a/llvm/test/CodeGen/AMDGPU/private-memory-atomics.ll
+++ b/llvm/test/CodeGen/AMDGPU/private-memory-atomics.ll
@@ -512,15 +512,16 @@ define amdgpu_kernel void @alloca_promote_atomicrmw_private_lds_promote(ptr addr
;
; GCN-LABEL: alloca_promote_atomicrmw_private_lds_promote:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s6, 1
-; GCN-NEXT: s_cselect_b32 s4, 1, 0
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_cmp_eq_u32 s2, 1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
entry:
%tmp = alloca [2 x i32], addrspace(5)
@@ -554,15 +555,16 @@ define amdgpu_kernel void @alloca_promote_cmpxchg_private(ptr addrspace(1) %out,
;
; GCN-LABEL: alloca_promote_cmpxchg_private:
; GCN: ; %bb.0: ; %entry
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_cmp_eq_u32 s6, 1
-; GCN-NEXT: s_cselect_b32 s4, 1, 0
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_cmp_eq_u32 s2, 1
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_cselect_b32 s0, 1, 0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
entry:
%tmp = alloca [2 x i32], addrspace(5)
diff --git a/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll b/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll
index ab6597363825e..836de5a8a6e75 100644
--- a/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll
+++ b/llvm/test/CodeGen/AMDGPU/promote-alloca-vector-dynamic-idx-bitcasts-llc.ll
@@ -7,8 +7,7 @@ define amdgpu_kernel void @test_bitcast_llc_v128i8_v16i8(ptr addrspace(1) %out,
; GFX9-LABEL: test_bitcast_llc_v128i8_v16i8:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_lshl_b32 s0, s0, 8
-; GFX9-NEXT: s_load_dwordx2 s[34:35], s[4:5], 0x0
-; GFX9-NEXT: s_load_dword s33, s[4:5], 0x8
+; GFX9-NEXT: s_load_dwordx4 s[36:39], s[4:5], 0x0
; GFX9-NEXT: s_and_b32 s1, s0, 0xff
; GFX9-NEXT: s_or_b32 s0, s1, s0
; GFX9-NEXT: s_and_b32 s1, s0, 0xffff
@@ -46,7 +45,7 @@ define amdgpu_kernel void @test_bitcast_llc_v128i8_v16i8(ptr addrspace(1) %out,
; GFX9-NEXT: s_mov_b32 s30, s0
; GFX9-NEXT: s_mov_b32 s31, s0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_lshl_b32 s33, s33, 2
+; GFX9-NEXT: s_lshl_b32 s33, s38, 2
; GFX9-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX9-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX9-NEXT: v_mov_b64_e32 v[4:5], s[4:5]
@@ -72,14 +71,12 @@ define amdgpu_kernel void @test_bitcast_llc_v128i8_v16i8(ptr addrspace(1) %out,
; GFX9-NEXT: v_mov_b32_e32 v33, v1
; GFX9-NEXT: v_mov_b32_e32 v32, v0
; GFX9-NEXT: s_set_gpr_idx_off
-; GFX9-NEXT: global_store_dwordx4 v36, v[32:35], s[34:35]
+; GFX9-NEXT: global_store_dwordx4 v36, v[32:35], s[36:37]
; GFX9-NEXT: s_endpgm
;
; GFX11-LABEL: test_bitcast_llc_v128i8_v16i8:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b64 s[34:35], s[4:5], 0x0
-; GFX11-NEXT: s_load_b32 s33, s[4:5], 0x8
+; GFX11-NEXT: s_load_b128 s[36:39], s[4:5], 0x0
; GFX11-NEXT: s_lshl_b32 s0, s0, 8
; GFX11-NEXT: v_mov_b32_e32 v35, 0
; GFX11-NEXT: s_and_b32 s1, s0, 0xff
@@ -123,7 +120,7 @@ define amdgpu_kernel void @test_bitcast_llc_v128i8_v16i8(ptr addrspace(1) %out,
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_lshl_b32 m0, s33, 2
+; GFX11-NEXT: s_lshl_b32 m0, s38, 2
; GFX11-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s5
; GFX11-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s7
@@ -142,7 +139,7 @@ define amdgpu_kernel void @test_bitcast_llc_v128i8_v16i8(ptr addrspace(1) %out,
; GFX11-NEXT: v_movrels_b32_e32 v33, v2
; GFX11-NEXT: v_movrels_b32_e32 v32, v1
; GFX11-NEXT: v_movrels_b32_e32 v31, v0
-; GFX11-NEXT: global_store_b128 v35, v[31:34], s[34:35]
+; GFX11-NEXT: global_store_b128 v35, v[31:34], s[36:37]
; GFX11-NEXT: s_endpgm
;
; GFX12-LABEL: test_bitcast_llc_v128i8_v16i8:
@@ -224,8 +221,7 @@ entry:
define amdgpu_kernel void @test_bitcast_llc_v64i16_v8i16(ptr addrspace(1) %out, i32 %idx) {
; GFX9-LABEL: test_bitcast_llc_v64i16_v8i16:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b32 s2, s2, 2
@@ -240,9 +236,7 @@ define amdgpu_kernel void @test_bitcast_llc_v64i16_v8i16(ptr addrspace(1) %out,
;
; GFX11-LABEL: test_bitcast_llc_v64i16_v8i16:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v4, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_lshl_b32 m0, s2, 2
@@ -277,8 +271,7 @@ entry:
define amdgpu_kernel void @test_bitcast_llc_v32i32_v4i32(ptr addrspace(1) %out, i32 %idx) {
; GFX9-LABEL: test_bitcast_llc_v32i32_v4i32:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v4, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b32 s2, s2, 2
@@ -293,9 +286,7 @@ define amdgpu_kernel void @test_bitcast_llc_v32i32_v4i32(ptr addrspace(1) %out,
;
; GFX11-LABEL: test_bitcast_llc_v32i32_v4i32:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v4, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_lshl_b32 m0, s2, 2
@@ -330,8 +321,7 @@ entry:
define amdgpu_kernel void @test_bitcast_llc_v16i64_v4i256(ptr addrspace(1) %out, i32 %idx) {
; GFX9-LABEL: test_bitcast_llc_v16i64_v4i256:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_lshl_b32 s2, s2, 3
@@ -351,9 +341,7 @@ define amdgpu_kernel void @test_bitcast_llc_v16i64_v4i256(ptr addrspace(1) %out,
;
; GFX11-LABEL: test_bitcast_llc_v16i64_v4i256:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x8
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; GFX11-NEXT: v_mov_b32_e32 v8, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_lshl_b32 m0, s2, 3
diff --git a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
index fba7720b37bf6..e87323b2be743 100644
--- a/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
+++ b/llvm/test/CodeGen/AMDGPU/ptradd-sdag-optimizations.ll
@@ -299,15 +299,14 @@ entry:
define amdgpu_kernel void @global_load_saddr_i32_uniform_offset(ptr addrspace(1) %sbase, i32 %soffset, ptr addrspace(1) %r) {
; GFX942-LABEL: global_load_saddr_i32_uniform_offset:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX942-NEXT: s_load_dword s6, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x10
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NEXT: s_load_dword s0, s[0:1], s6 offset:0x0
+; GFX942-NEXT: s_load_dword s0, s[0:1], s2 offset:0x0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_mov_b32_e32 v1, s0
-; GFX942-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX942-NEXT: global_store_dword v0, v1, s[6:7]
; GFX942-NEXT: s_endpgm
%zext.offset = zext i32 %soffset to i64
%gep0 = getelementptr inbounds i8, ptr addrspace(1) %sbase, i64 %zext.offset
diff --git a/llvm/test/CodeGen/AMDGPU/rcp-pattern.ll b/llvm/test/CodeGen/AMDGPU/rcp-pattern.ll
index 26f2b3d2af60d..1f9ca1e3bdbff 100644
--- a/llvm/test/CodeGen/AMDGPU/rcp-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/rcp-pattern.ll
@@ -748,10 +748,9 @@ define float @v_rcp_neg_fabs_f32_daz_ulp25(float %x) #0 {
define amdgpu_kernel void @s_rcp_pat_f32_daz(ptr addrspace(1) %out, float %src) #0 {
; SI-LABEL: s_rcp_pat_f32_daz:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: v_rcp_f32_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -759,8 +758,7 @@ define amdgpu_kernel void @s_rcp_pat_f32_daz(ptr addrspace(1) %out, float %src)
;
; VI-LABEL: s_rcp_pat_f32_daz:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_rcp_f32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -800,10 +798,9 @@ define amdgpu_kernel void @s_rcp_pat_f32_daz(ptr addrspace(1) %out, float %src)
define amdgpu_kernel void @s_rcp_ulp25_pat_f32_daz(ptr addrspace(1) %out, float %src) #0 {
; SI-LABEL: s_rcp_ulp25_pat_f32_daz:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: v_rcp_f32_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -811,8 +808,7 @@ define amdgpu_kernel void @s_rcp_ulp25_pat_f32_daz(ptr addrspace(1) %out, float
;
; VI-LABEL: s_rcp_ulp25_pat_f32_daz:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_rcp_f32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -852,10 +848,9 @@ define amdgpu_kernel void @s_rcp_ulp25_pat_f32_daz(ptr addrspace(1) %out, float
define amdgpu_kernel void @s_rcp_fast_ulp25_pat_f32_daz(ptr addrspace(1) %out, float %src) #0 {
; SI-LABEL: s_rcp_fast_ulp25_pat_f32_daz:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: v_rcp_f32_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -863,8 +858,7 @@ define amdgpu_kernel void @s_rcp_fast_ulp25_pat_f32_daz(ptr addrspace(1) %out, f
;
; VI-LABEL: s_rcp_fast_ulp25_pat_f32_daz:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_rcp_f32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -904,10 +898,9 @@ define amdgpu_kernel void @s_rcp_fast_ulp25_pat_f32_daz(ptr addrspace(1) %out, f
define amdgpu_kernel void @s_rcp_arcp_ulp25_pat_f32_daz(ptr addrspace(1) %out, float %src) #0 {
; SI-LABEL: s_rcp_arcp_ulp25_pat_f32_daz:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: v_rcp_f32_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -915,8 +908,7 @@ define amdgpu_kernel void @s_rcp_arcp_ulp25_pat_f32_daz(ptr addrspace(1) %out, f
;
; VI-LABEL: s_rcp_arcp_ulp25_pat_f32_daz:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_rcp_f32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -956,10 +948,9 @@ define amdgpu_kernel void @s_rcp_arcp_ulp25_pat_f32_daz(ptr addrspace(1) %out, f
define amdgpu_kernel void @s_rcp_global_fast_ulp25_pat_f32_daz(ptr addrspace(1) %out, float %src) #2 {
; SI-LABEL: s_rcp_global_fast_ulp25_pat_f32_daz:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: v_rcp_f32_e32 v0, s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -967,8 +958,7 @@ define amdgpu_kernel void @s_rcp_global_fast_ulp25_pat_f32_daz(ptr addrspace(1)
;
; VI-LABEL: s_rcp_global_fast_ulp25_pat_f32_daz:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_rcp_f32_e32 v2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -1008,10 +998,9 @@ define amdgpu_kernel void @s_rcp_global_fast_ulp25_pat_f32_daz(ptr addrspace(1)
define amdgpu_kernel void @s_rcp_fabs_pat_f32_daz(ptr addrspace(1) %out, float %src) #0 {
; SI-LABEL: s_rcp_fabs_pat_f32_daz:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: v_rcp_f32_e64 v0, |s2|
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -1019,8 +1008,7 @@ define amdgpu_kernel void @s_rcp_fabs_pat_f32_daz(ptr addrspace(1) %out, float %
;
; VI-LABEL: s_rcp_fabs_pat_f32_daz:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_rcp_f32_e64 v2, |s2|
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -1061,10 +1049,9 @@ define amdgpu_kernel void @s_rcp_fabs_pat_f32_daz(ptr addrspace(1) %out, float %
define amdgpu_kernel void @s_neg_rcp_pat_f32_daz(ptr addrspace(1) %out, float %src) #0 {
; SI-LABEL: s_neg_rcp_pat_f32_daz:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: v_rcp_f32_e64 v0, -s2
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -1072,8 +1059,7 @@ define amdgpu_kernel void @s_neg_rcp_pat_f32_daz(ptr addrspace(1) %out, float %s
;
; VI-LABEL: s_neg_rcp_pat_f32_daz:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_rcp_f32_e64 v2, -s2
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -1116,10 +1102,9 @@ define amdgpu_kernel void @s_neg_rcp_pat_f32_daz(ptr addrspace(1) %out, float %s
define amdgpu_kernel void @s_rcp_fabs_fneg_pat_f32_daz(ptr addrspace(1) %out, float %src) #0 {
; SI-LABEL: s_rcp_fabs_fneg_pat_f32_daz:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: v_rcp_f32_e64 v0, -|s2|
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -1127,8 +1112,7 @@ define amdgpu_kernel void @s_rcp_fabs_fneg_pat_f32_daz(ptr addrspace(1) %out, fl
;
; VI-LABEL: s_rcp_fabs_fneg_pat_f32_daz:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_rcp_f32_e64 v2, -|s2|
; VI-NEXT: v_mov_b32_e32 v0, s0
@@ -1173,23 +1157,23 @@ define amdgpu_kernel void @s_rcp_fabs_fneg_pat_f32_daz(ptr addrspace(1) %out, fl
define amdgpu_kernel void @s_rcp_fabs_fneg_pat_multi_use_f32_daz(ptr addrspace(1) %out, float %src) #0 {
; SI-LABEL: s_rcp_fabs_fneg_pat_multi_use_f32_daz:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_rcp_f32_e64 v0, -|s6|
-; SI-NEXT: v_mul_f32_e64 v1, s6, -|s6|
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: v_rcp_f32_e64 v0, -|s2|
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mul_f32_e64 v1, s2, -|s2|
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: buffer_store_dword v1, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v1, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_rcp_fabs_fneg_pat_multi_use_f32_daz:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_rcp_f32_e64 v2, -|s2|
; VI-NEXT: v_mov_b32_e32 v0, s0
diff --git a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
index 74bd338c7c022..7f274ef54153a 100644
--- a/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
+++ b/llvm/test/CodeGen/AMDGPU/reassoc-mul-add-1-to-mad.ll
@@ -5102,69 +5102,63 @@ define i64 @v_mul_934584645_add_8234599_i64(i64 %arg) {
define amdgpu_kernel void @compute_mad(ptr addrspace(4) %i18, ptr addrspace(4) %i21, ptr addrspace(1) nocapture noundef writeonly align 4 %arg, i32 noundef %arg1) #1 {
; GFX67-LABEL: compute_mad:
; GFX67: ; %bb.0: ; %bb
-; GFX67-NEXT: s_load_dword s0, s[4:5], 0x6
-; GFX67-NEXT: s_mov_b32 s3, 0xf000
+; GFX67-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x0
; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: s_add_i32 s0, s0, 1
-; GFX67-NEXT: v_mul_lo_u32 v1, s0, v0
-; GFX67-NEXT: v_add_i32_e32 v2, vcc, s0, v1
+; GFX67-NEXT: s_mov_b32 s7, 0xf000
+; GFX67-NEXT: s_load_dword s2, s[2:3], 0x1
+; GFX67-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
+; GFX67-NEXT: s_add_i32 s6, s6, 1
+; GFX67-NEXT: v_mul_lo_u32 v1, s6, v0
+; GFX67-NEXT: s_waitcnt lgkmcnt(0)
+; GFX67-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX67-NEXT: s_mul_i32 s8, s8, s2
+; GFX67-NEXT: v_add_i32_e32 v2, vcc, s6, v1
; GFX67-NEXT: v_mul_lo_u32 v2, v2, v0
; GFX67-NEXT: v_add_i32_e32 v1, vcc, 1, v1
-; GFX67-NEXT: s_load_dwordx4 s[12:15], s[4:5], 0x0
-; GFX67-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x4
+; GFX67-NEXT: v_add_i32_e32 v0, vcc, s8, v0
; GFX67-NEXT: v_mul_lo_u32 v3, v2, v1
-; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: s_load_dword s2, s[14:15], 0x1
-; GFX67-NEXT: s_load_dwordx2 s[4:5], s[12:13], 0x0
+; GFX67-NEXT: s_mov_b32 s6, 0
; GFX67-NEXT: v_add_i32_e32 v1, vcc, v3, v1
; GFX67-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX67-NEXT: v_add_i32_e32 v2, vcc, 1, v3
-; GFX67-NEXT: s_waitcnt lgkmcnt(0)
-; GFX67-NEXT: s_and_b32 s2, s2, 0xffff
; GFX67-NEXT: v_mul_lo_u32 v3, v1, v2
-; GFX67-NEXT: s_mul_i32 s8, s8, s2
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, s8, v0
; GFX67-NEXT: v_add_i32_e32 v2, vcc, v3, v2
; GFX67-NEXT: v_mul_lo_u32 v1, v2, v1
-; GFX67-NEXT: v_mov_b32_e32 v2, s5
-; GFX67-NEXT: s_mov_b32 s2, 0
+; GFX67-NEXT: v_mov_b32_e32 v2, s1
; GFX67-NEXT: v_mul_lo_u32 v3, v1, v3
; GFX67-NEXT: v_add_i32_e32 v3, vcc, v3, v1
; GFX67-NEXT: v_mul_lo_u32 v4, v3, v1
-; GFX67-NEXT: v_add_i32_e32 v0, vcc, s4, v0
+; GFX67-NEXT: v_add_i32_e32 v0, vcc, s0, v0
; GFX67-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
; GFX67-NEXT: v_lshl_b64 v[0:1], v[0:1], 2
; GFX67-NEXT: v_add_i32_e32 v2, vcc, v4, v3
-; GFX67-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; GFX67-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; GFX67-NEXT: s_endpgm
;
; GFX8-LABEL: compute_mad:
; GFX8: ; %bb.0: ; %bb
-; GFX8-NEXT: s_load_dword s0, s[4:5], 0x18
+; GFX8-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_add_i32 s0, s0, 1
-; GFX8-NEXT: v_mul_lo_u32 v1, s0, v0
-; GFX8-NEXT: v_add_u32_e32 v2, vcc, s0, v1
+; GFX8-NEXT: s_add_i32 s6, s6, 1
+; GFX8-NEXT: v_mul_lo_u32 v1, s6, v0
+; GFX8-NEXT: s_load_dword s2, s[2:3], 0x4
+; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
+; GFX8-NEXT: v_mov_b32_e32 v4, s5
+; GFX8-NEXT: v_add_u32_e32 v2, vcc, s6, v1
; GFX8-NEXT: v_mul_lo_u32 v2, v2, v0
; GFX8-NEXT: v_add_u32_e32 v1, vcc, 1, v1
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x10
-; GFX8-NEXT: v_mul_lo_u32 v3, v2, v1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_load_dword s2, s[2:3], 0x4
+; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX8-NEXT: v_mul_lo_u32 v3, v2, v1
+; GFX8-NEXT: s_mul_i32 s8, s8, s2
+; GFX8-NEXT: v_add_u32_e32 v0, vcc, s8, v0
; GFX8-NEXT: v_add_u32_e32 v1, vcc, v3, v1
; GFX8-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX8-NEXT: v_add_u32_e32 v2, vcc, 1, v3
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX8-NEXT: v_mul_lo_u32 v3, v1, v2
-; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX8-NEXT: s_mul_i32 s8, s8, s2
-; GFX8-NEXT: v_add_u32_e32 v0, vcc, s8, v0
; GFX8-NEXT: v_add_u32_e32 v2, vcc, v3, v2
; GFX8-NEXT: v_mul_lo_u32 v1, v2, v1
; GFX8-NEXT: v_mov_b32_e32 v2, s1
-; GFX8-NEXT: v_mov_b32_e32 v4, s5
; GFX8-NEXT: v_mul_lo_u32 v3, v1, v3
; GFX8-NEXT: v_add_u32_e32 v3, vcc, v3, v1
; GFX8-NEXT: v_mul_lo_u32 v5, v3, v1
@@ -5179,29 +5173,26 @@ define amdgpu_kernel void @compute_mad(ptr addrspace(4) %i18, ptr addrspace(4) %
;
; GFX900-LABEL: compute_mad:
; GFX900: ; %bb.0: ; %bb
-; GFX900-NEXT: s_load_dword s0, s[4:5], 0x18
+; GFX900-NEXT: s_load_dwordx8 s[12:19], s[4:5], 0x0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: s_add_i32 s0, s0, 1
+; GFX900-NEXT: s_add_i32 s0, s18, 1
; GFX900-NEXT: v_mul_lo_u32 v1, s0, v0
+; GFX900-NEXT: s_load_dword s2, s[14:15], 0x4
+; GFX900-NEXT: v_mov_b32_e32 v5, s17
; GFX900-NEXT: v_add_u32_e32 v2, s0, v1
; GFX900-NEXT: v_mul_lo_u32 v2, v2, v0
; GFX900-NEXT: v_add_u32_e32 v1, 1, v1
-; GFX900-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX900-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX900-NEXT: s_load_dwordx2 s[0:1], s[12:13], 0x0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: s_load_dword s9, s[2:3], 0x4
+; GFX900-NEXT: s_and_b32 s2, s2, 0xffff
; GFX900-NEXT: v_mul_lo_u32 v3, v2, v1
-; GFX900-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
-; GFX900-NEXT: v_mov_b32_e32 v5, s7
+; GFX900-NEXT: s_mul_i32 s8, s8, s2
+; GFX900-NEXT: v_add_u32_e32 v0, s8, v0
+; GFX900-NEXT: v_mov_b32_e32 v4, s1
; GFX900-NEXT: v_add_u32_e32 v1, v3, v1
; GFX900-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX900-NEXT: v_add_u32_e32 v2, 1, v3
-; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: s_and_b32 s2, s9, 0xffff
-; GFX900-NEXT: s_mul_i32 s8, s8, s2
; GFX900-NEXT: v_mul_lo_u32 v3, v1, v2
-; GFX900-NEXT: v_add_u32_e32 v0, s8, v0
-; GFX900-NEXT: v_mov_b32_e32 v4, s1
; GFX900-NEXT: v_add_u32_e32 v2, v3, v2
; GFX900-NEXT: v_mul_lo_u32 v1, v2, v1
; GFX900-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v1, v3, v[1:2]
@@ -5209,33 +5200,31 @@ define amdgpu_kernel void @compute_mad(ptr addrspace(4) %i18, ptr addrspace(4) %
; GFX900-NEXT: v_addc_co_u32_e32 v4, vcc, 0, v4, vcc
; GFX900-NEXT: v_lshlrev_b64 v[3:4], 2, v[3:4]
; GFX900-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, v1, v[2:3]
-; GFX900-NEXT: v_add_co_u32_e32 v1, vcc, s6, v3
+; GFX900-NEXT: v_add_co_u32_e32 v1, vcc, s16, v3
; GFX900-NEXT: v_addc_co_u32_e32 v2, vcc, v5, v4, vcc
; GFX900-NEXT: global_store_dword v[1:2], v0, off
; GFX900-NEXT: s_endpgm
;
; GFX90A-LABEL: compute_mad:
; GFX90A: ; %bb.0: ; %bb
-; GFX90A-NEXT: s_load_dword s9, s[4:5], 0x18
-; GFX90A-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
-; GFX90A-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x10
+; GFX90A-NEXT: s_load_dwordx8 s[12:19], s[4:5], 0x0
; GFX90A-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NEXT: s_add_i32 s9, s9, 1
-; GFX90A-NEXT: v_mul_lo_u32 v0, s9, v4
-; GFX90A-NEXT: v_add_u32_e32 v1, s9, v0
-; GFX90A-NEXT: v_mul_lo_u32 v1, v1, v4
+; GFX90A-NEXT: s_add_i32 s0, s18, 1
+; GFX90A-NEXT: v_mul_lo_u32 v0, s0, v4
+; GFX90A-NEXT: v_add_u32_e32 v1, s0, v0
; GFX90A-NEXT: v_add_u32_e32 v0, 1, v0
+; GFX90A-NEXT: v_mul_lo_u32 v1, v1, v4
; GFX90A-NEXT: v_mul_lo_u32 v2, v1, v0
; GFX90A-NEXT: v_add_u32_e32 v0, v2, v0
; GFX90A-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX90A-NEXT: v_add_u32_e32 v1, 1, v2
; GFX90A-NEXT: v_mul_lo_u32 v2, v0, v1
-; GFX90A-NEXT: s_load_dword s4, s[2:3], 0x4
; GFX90A-NEXT: v_add_u32_e32 v1, v2, v1
+; GFX90A-NEXT: s_load_dword s4, s[14:15], 0x4
; GFX90A-NEXT: v_mul_lo_u32 v0, v1, v0
-; GFX90A-NEXT: v_mad_u64_u32 v[2:3], s[2:3], v0, v2, v[0:1]
-; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
+; GFX90A-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v0, v2, v[0:1]
+; GFX90A-NEXT: s_load_dwordx2 s[0:1], s[12:13], 0x0
; GFX90A-NEXT: v_mad_u64_u32 v[0:1], s[2:3], v2, v0, v[2:3]
; GFX90A-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NEXT: s_and_b32 s2, s4, 0xffff
@@ -5245,33 +5234,30 @@ define amdgpu_kernel void @compute_mad(ptr addrspace(4) %i18, ptr addrspace(4) %
; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, s0, v1
; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
; GFX90A-NEXT: v_lshlrev_b64 v[2:3], 2, v[2:3]
-; GFX90A-NEXT: v_mov_b32_e32 v1, s7
-; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, s6, v2
+; GFX90A-NEXT: v_mov_b32_e32 v1, s17
+; GFX90A-NEXT: v_add_co_u32_e32 v2, vcc, s16, v2
; GFX90A-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
; GFX90A-NEXT: global_store_dword v[2:3], v0, off
; GFX90A-NEXT: s_endpgm
;
; GFX10-LABEL: compute_mad:
; GFX10: ; %bb.0: ; %bb
-; GFX10-NEXT: s_load_dword s0, s[4:5], 0x18
+; GFX10-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_add_i32 s0, s0, 1
-; GFX10-NEXT: v_mul_lo_u32 v1, s0, v0
-; GFX10-NEXT: v_add_nc_u32_e32 v2, s0, v1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX10-NEXT: s_add_i32 s6, s6, 1
+; GFX10-NEXT: s_load_dword s2, s[2:3], 0x4
+; GFX10-NEXT: v_mul_lo_u32 v1, s6, v0
+; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
+; GFX10-NEXT: v_add_nc_u32_e32 v2, s6, v1
; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v1
-; GFX10-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x10
; GFX10-NEXT: v_mul_lo_u32 v2, v2, v0
-; GFX10-NEXT: v_mul_lo_u32 v3, v2, v1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_load_dword s2, s[2:3], 0x4
+; GFX10-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX10-NEXT: v_mul_lo_u32 v3, v2, v1
; GFX10-NEXT: v_add_nc_u32_e32 v1, v3, v1
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x0
; GFX10-NEXT: v_mul_lo_u32 v2, v1, v2
; GFX10-NEXT: v_add_nc_u32_e32 v1, 1, v3
; GFX10-NEXT: v_mul_lo_u32 v4, v2, v1
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_and_b32 s2, s2, 0xffff
; GFX10-NEXT: v_add_nc_u32_e32 v3, v4, v1
; GFX10-NEXT: v_mad_u64_u32 v[0:1], null, s8, s2, v[0:1]
; GFX10-NEXT: v_mul_lo_u32 v1, v3, v2
@@ -5288,47 +5274,48 @@ define amdgpu_kernel void @compute_mad(ptr addrspace(4) %i18, ptr addrspace(4) %
; GFX1250-LABEL: compute_mad:
; GFX1250: ; %bb.0: ; %bb
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x10 nv
+; GFX1250-NEXT: s_load_b96 s[8:10], s[4:5], 0x10 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_load_b128 s[4:7], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_add_co_i32 s2, s2, 1
-; GFX1250-NEXT: s_load_b32 s6, s[6:7], 0x4 nv
-; GFX1250-NEXT: v_mul_lo_u32 v1, s2, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_dual_add_nc_u32 v2, s2, v1 :: v_dual_add_nc_u32 v1, 1, v1
-; GFX1250-NEXT: s_bfe_u32 s2, ttmp6, 0x4000c
-; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_add_co_i32 s7, s2, 1
-; GFX1250-NEXT: v_mul_lo_u32 v2, v2, v0
-; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_add_co_i32 s0, s10, 1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: v_mul_lo_u32 v1, s0, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_add_nc_u32 v2, s0, v1 :: v_dual_add_nc_u32 v1, 1, v1
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: s_and_b32 s4, ttmp6, 15
-; GFX1250-NEXT: s_mul_i32 s5, ttmp9, s7
-; GFX1250-NEXT: s_getreg_b32 s7, hwreg(HW_REG_IB_STS2, 6, 4)
-; GFX1250-NEXT: s_add_co_i32 s4, s4, s5
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_and_b32 s5, s6, 0xffff
-; GFX1250-NEXT: s_cmp_eq_u32 s7, 0
+; GFX1250-NEXT: s_getreg_b32 s5, hwreg(HW_REG_IB_STS2, 6, 4)
+; GFX1250-NEXT: v_mul_lo_u32 v2, v2, v0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_mul_lo_u32 v3, v2, v1
-; GFX1250-NEXT: s_cselect_b32 s4, ttmp9, s4
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_mad_u32 v0, s4, s5, v0
; GFX1250-NEXT: v_add_nc_u32_e32 v1, v3, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s2, s[2:3], 0x4 nv
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: s_bfe_u32 s3, ttmp6, 0x4000c
+; GFX1250-NEXT: s_load_b64 s[0:1], s[0:1], 0x0 nv
+; GFX1250-NEXT: s_add_co_i32 s3, s3, 1
; GFX1250-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX1250-NEXT: v_add_nc_u32_e32 v2, 1, v3
+; GFX1250-NEXT: s_mul_i32 s3, ttmp9, s3
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_add_co_i32 s4, s4, s3
; GFX1250-NEXT: v_mul_lo_u32 v3, v1, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_add_nc_u32_e32 v2, v3, v2
+; GFX1250-NEXT: s_cmp_eq_u32 s5, 0
+; GFX1250-NEXT: s_cselect_b32 s3, ttmp9, s4
; GFX1250-NEXT: v_mul_lo_u32 v2, v2, v1
+; GFX1250-NEXT: v_mad_u32 v0, s3, s2, v0
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_mad_u32 v3, v2, v3, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[0:1]
+; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], s[0:1], v[0:1]
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_lshl_add_u64 v[0:1], v[0:1], 2, s[8:9]
; GFX1250-NEXT: v_mad_u32 v2, v3, v2, v3
; GFX1250-NEXT: global_store_b32 v[0:1], v2, off
; GFX1250-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/reg-coalescer-sched-crash.ll b/llvm/test/CodeGen/AMDGPU/reg-coalescer-sched-crash.ll
index a3b3831f5b954..aa68e54e76375 100644
--- a/llvm/test/CodeGen/AMDGPU/reg-coalescer-sched-crash.ll
+++ b/llvm/test/CodeGen/AMDGPU/reg-coalescer-sched-crash.ll
@@ -16,13 +16,12 @@ define amdgpu_kernel void @reg_coalescer_breaks_dead(ptr addrspace(1) nocapture
; GFX6-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX6-NEXT: s_cbranch_execz .LBB0_2
; GFX6-NEXT: ; %bb.1: ; %bb3
-; GFX6-NEXT: s_load_dword s2, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x9
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_ashr_i32 s3, s2, 31
-; GFX6-NEXT: s_lshl_b64 s[2:3], s[2:3], 3
-; GFX6-NEXT: s_add_u32 s2, s6, s2
-; GFX6-NEXT: s_addc_u32 s3, s7, s3
+; GFX6-NEXT: s_ashr_i32 s11, s10, 31
+; GFX6-NEXT: s_lshl_b64 s[2:3], s[10:11], 3
+; GFX6-NEXT: s_add_u32 s2, s8, s2
+; GFX6-NEXT: s_addc_u32 s3, s9, s3
; GFX6-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v1, s2
@@ -46,13 +45,12 @@ define amdgpu_kernel void @reg_coalescer_breaks_dead(ptr addrspace(1) nocapture
; GFX8-NEXT: s_and_saveexec_b64 s[0:1], vcc
; GFX8-NEXT: s_cbranch_execz .LBB0_2
; GFX8-NEXT: ; %bb.1: ; %bb3
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_ashr_i32 s3, s2, 31
-; GFX8-NEXT: s_lshl_b64 s[2:3], s[2:3], 3
-; GFX8-NEXT: s_add_u32 s2, s6, s2
-; GFX8-NEXT: s_addc_u32 s3, s7, s3
+; GFX8-NEXT: s_ashr_i32 s11, s10, 31
+; GFX8-NEXT: s_lshl_b64 s[2:3], s[10:11], 3
+; GFX8-NEXT: s_add_u32 s2, s8, s2
+; GFX8-NEXT: s_addc_u32 s3, s9, s3
; GFX8-NEXT: s_load_dwordx2 s[2:3], s[2:3], 0x0
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v1, s2
diff --git a/llvm/test/CodeGen/AMDGPU/returnaddress.ll b/llvm/test/CodeGen/AMDGPU/returnaddress.ll
index f39d29f4bfefe..9bca0326c6659 100644
--- a/llvm/test/CodeGen/AMDGPU/returnaddress.ll
+++ b/llvm/test/CodeGen/AMDGPU/returnaddress.ll
@@ -36,8 +36,7 @@ entry:
; Test with use outside the entry-block
; GCN-LABEL: {{^}}func4
-; GCN: v_mov_b32_e32 v0, 0
-; GCN: v_mov_b32_e32 v1, {{v0|0}}
+; GCN: flat_store_dwordx2
define amdgpu_kernel void @func4(ptr %out, i32 %val) nounwind {
entry:
%cmp = icmp ne i32 %val, 0
diff --git a/llvm/test/CodeGen/AMDGPU/rotl.ll b/llvm/test/CodeGen/AMDGPU/rotl.ll
index a7fcb6439703a..4192c90f0105d 100644
--- a/llvm/test/CodeGen/AMDGPU/rotl.ll
+++ b/llvm/test/CodeGen/AMDGPU/rotl.ll
@@ -40,13 +40,13 @@ define amdgpu_kernel void @rotl_i32(ptr addrspace(1) %in, i32 %x, i32 %y) {
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sub_i32 s4, 32, s3
-; GFX8-NEXT: s_mov_b32 s3, s2
-; GFX8-NEXT: s_and_b32 s4, s4, 31
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_sub_i32 s0, 32, s3
+; GFX8-NEXT: s_and_b32 s0, s0, 31
+; GFX8-NEXT: s_mov_b32 s3, s2
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -104,86 +104,88 @@ define amdgpu_kernel void @rotl_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
;
; SI-LABEL: rotl_v2i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_sub_i32 s6, 32, s2
-; SI-NEXT: s_sub_i32 s8, 32, s3
-; SI-NEXT: s_mov_b32 s2, s1
-; SI-NEXT: s_mov_b32 s3, s1
-; SI-NEXT: s_mov_b32 s1, s0
+; SI-NEXT: s_sub_i32 s6, 32, s8
+; SI-NEXT: s_sub_i32 s8, 32, s9
+; SI-NEXT: s_mov_b32 s4, s3
+; SI-NEXT: s_mov_b32 s5, s3
+; SI-NEXT: s_mov_b32 s3, s2
; SI-NEXT: s_and_b32 s8, s8, 31
; SI-NEXT: s_and_b32 s6, s6, 31
-; SI-NEXT: s_lshr_b64 s[2:3], s[2:3], s8
-; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], s6
+; SI-NEXT: s_lshr_b64 s[8:9], s[4:5], s8
+; SI-NEXT: s_lshr_b64 s[2:3], s[2:3], s6
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s2
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; GFX8-LABEL: rotl_v2i32:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sub_i32 s7, 32, s3
-; GFX8-NEXT: s_sub_i32 s6, 32, s2
-; GFX8-NEXT: s_mov_b32 s2, s1
-; GFX8-NEXT: s_mov_b32 s3, s1
-; GFX8-NEXT: s_and_b32 s1, s7, 31
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s1
-; GFX8-NEXT: s_and_b32 s3, s6, 31
-; GFX8-NEXT: s_mov_b32 s1, s0
-; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s3
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s5
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: s_sub_i32 s0, 32, s5
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: s_and_b32 s5, s0, 31
+; GFX8-NEXT: s_mov_b32 s0, s3
+; GFX8-NEXT: s_mov_b32 s1, s3
+; GFX8-NEXT: s_sub_i32 s4, 32, s4
+; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s5
+; GFX8-NEXT: s_and_b32 s1, s4, 31
+; GFX8-NEXT: s_mov_b32 s3, s2
+; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX10-LABEL: rotl_v2i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_sub_i32 s4, 32, s2
-; GFX10-NEXT: s_sub_i32 s5, 32, s3
-; GFX10-NEXT: s_mov_b32 s2, s1
-; GFX10-NEXT: s_mov_b32 s3, s1
-; GFX10-NEXT: s_mov_b32 s1, s0
-; GFX10-NEXT: s_and_b32 s4, s4, 31
-; GFX10-NEXT: s_and_b32 s5, s5, 31
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s4
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s5
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX10-NEXT: s_sub_i32 s6, 32, s6
+; GFX10-NEXT: s_sub_i32 s7, 32, s7
+; GFX10-NEXT: s_mov_b32 s4, s3
+; GFX10-NEXT: s_mov_b32 s5, s3
+; GFX10-NEXT: s_mov_b32 s3, s2
+; GFX10-NEXT: s_and_b32 s6, s6, 31
+; GFX10-NEXT: s_and_b32 s7, s7, 31
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s6
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s7
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: rotl_v2i32:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_sub_i32 s6, 32, s2
-; GFX11-NEXT: s_sub_i32 s7, 32, s3
-; GFX11-NEXT: s_mov_b32 s2, s1
-; GFX11-NEXT: s_mov_b32 s3, s1
-; GFX11-NEXT: s_mov_b32 s1, s0
+; GFX11-NEXT: s_sub_i32 s6, 32, s6
+; GFX11-NEXT: s_sub_i32 s7, 32, s7
+; GFX11-NEXT: s_mov_b32 s4, s3
+; GFX11-NEXT: s_mov_b32 s5, s3
+; GFX11-NEXT: s_mov_b32 s3, s2
; GFX11-NEXT: s_and_b32 s6, s6, 31
; GFX11-NEXT: s_and_b32 s7, s7, 31
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s6
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s7
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s6
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[4:5], s7
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
entry:
%0 = shl <2 x i32> %x, %y
diff --git a/llvm/test/CodeGen/AMDGPU/rotr.ll b/llvm/test/CodeGen/AMDGPU/rotr.ll
index 581312dd3e73f..460717c9f052f 100644
--- a/llvm/test/CodeGen/AMDGPU/rotr.ll
+++ b/llvm/test/CodeGen/AMDGPU/rotr.ll
@@ -37,12 +37,12 @@ define amdgpu_kernel void @rotr_i32(ptr addrspace(1) %in, i32 %x, i32 %y) {
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s4, s3, 31
-; GFX8-NEXT: s_mov_b32 s3, s2
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_and_b32 s0, s3, 31
+; GFX8-NEXT: s_mov_b32 s3, s2
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: s_lshr_b64 s[0:1], s[2:3], s0
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -93,78 +93,80 @@ define amdgpu_kernel void @rotr_v2i32(ptr addrspace(1) %in, <2 x i32> %x, <2 x i
;
; SI-LABEL: rotr_v2i32:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s3, s3, 31
-; SI-NEXT: s_mov_b32 s8, s1
-; SI-NEXT: s_mov_b32 s9, s1
-; SI-NEXT: s_and_b32 s6, s2, 31
-; SI-NEXT: s_mov_b32 s1, s0
-; SI-NEXT: s_lshr_b64 s[2:3], s[8:9], s3
-; SI-NEXT: s_lshr_b64 s[0:1], s[0:1], s6
+; SI-NEXT: s_and_b32 s5, s5, 31
+; SI-NEXT: s_mov_b32 s8, s3
+; SI-NEXT: s_mov_b32 s9, s3
+; SI-NEXT: s_and_b32 s4, s4, 31
+; SI-NEXT: s_mov_b32 s3, s2
+; SI-NEXT: s_lshr_b64 s[8:9], s[8:9], s5
+; SI-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: v_mov_b32_e32 v1, s2
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s8
; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; GFX8-LABEL: rotr_v2i32:
; GFX8: ; %bb.0: ; %entry
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x34
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_and_b32 s3, s3, 31
-; GFX8-NEXT: s_mov_b32 s6, s1
-; GFX8-NEXT: s_mov_b32 s7, s1
-; GFX8-NEXT: s_and_b32 s8, s2, 31
-; GFX8-NEXT: s_mov_b32 s1, s0
-; GFX8-NEXT: s_lshr_b64 s[2:3], s[6:7], s3
-; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s8
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s2
-; GFX8-NEXT: v_mov_b32_e32 v3, s5
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: s_and_b32 s5, s5, 31
+; GFX8-NEXT: s_mov_b32 s0, s3
+; GFX8-NEXT: s_mov_b32 s1, s3
+; GFX8-NEXT: s_lshr_b64 s[0:1], s[0:1], s5
+; GFX8-NEXT: s_and_b32 s1, s4, 31
+; GFX8-NEXT: s_mov_b32 s3, s2
+; GFX8-NEXT: s_lshr_b64 s[2:3], s[2:3], s1
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
;
; GFX10-LABEL: rotr_v2i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_mov_b32 s4, s1
-; GFX10-NEXT: s_mov_b32 s5, s1
-; GFX10-NEXT: s_and_b32 s2, s2, 31
-; GFX10-NEXT: s_mov_b32 s1, s0
-; GFX10-NEXT: s_and_b32 s3, s3, 31
-; GFX10-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX10-NEXT: s_lshr_b64 s[2:3], s[4:5], s3
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s2
-; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[6:7]
+; GFX10-NEXT: s_mov_b32 s4, s3
+; GFX10-NEXT: s_mov_b32 s5, s3
+; GFX10-NEXT: s_and_b32 s6, s6, 31
+; GFX10-NEXT: s_mov_b32 s3, s2
+; GFX10-NEXT: s_and_b32 s7, s7, 31
+; GFX10-NEXT: s_lshr_b64 s[2:3], s[2:3], s6
+; GFX10-NEXT: s_lshr_b64 s[4:5], s[4:5], s7
+; GFX10-NEXT: v_mov_b32_e32 v0, s2
+; GFX10-NEXT: v_mov_b32_e32 v1, s4
+; GFX10-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: rotr_v2i32:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_mov_b32 s6, s1
-; GFX11-NEXT: s_mov_b32 s7, s1
-; GFX11-NEXT: s_and_b32 s2, s2, 31
-; GFX11-NEXT: s_mov_b32 s1, s0
-; GFX11-NEXT: s_and_b32 s3, s3, 31
-; GFX11-NEXT: s_lshr_b64 s[0:1], s[0:1], s2
-; GFX11-NEXT: s_lshr_b64 s[2:3], s[6:7], s3
+; GFX11-NEXT: s_mov_b32 s6, s3
+; GFX11-NEXT: s_mov_b32 s7, s3
+; GFX11-NEXT: s_and_b32 s4, s4, 31
+; GFX11-NEXT: s_mov_b32 s3, s2
+; GFX11-NEXT: s_and_b32 s5, s5, 31
+; GFX11-NEXT: s_lshr_b64 s[2:3], s[2:3], s4
+; GFX11-NEXT: s_lshr_b64 s[4:5], s[6:7], s5
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
-; GFX11-NEXT: v_mov_b32_e32 v0, s0
-; GFX11-NEXT: global_store_b64 v2, v[0:1], s[4:5]
+; GFX11-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_mov_b32_e32 v0, s2
+; GFX11-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX11-NEXT: s_endpgm
entry:
%tmp0 = sub <2 x i32> <i32 32, i32 32>, %y
diff --git a/llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll b/llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll
index 8c9ebf8d23d9f..62b4ec33ba2e2 100644
--- a/llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll
+++ b/llvm/test/CodeGen/AMDGPU/rsq.f32-safe.ll
@@ -64,10 +64,9 @@ define amdgpu_kernel void @rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(
define amdgpu_kernel void @rsq_f32_sgpr(ptr addrspace(1) noalias %out, float %val) {
; GCN-DAZ-LABEL: rsq_f32_sgpr:
; GCN-DAZ: ; %bb.0:
-; GCN-DAZ-NEXT: s_load_dword s2, s[4:5], 0xb
-; GCN-DAZ-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-DAZ-NEXT: s_mov_b32 s3, 0xf000
+; GCN-DAZ-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-DAZ-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT: s_mov_b32 s3, 0xf000
; GCN-DAZ-NEXT: v_rsq_f32_e32 v0, s2
; GCN-DAZ-NEXT: s_mov_b32 s2, -1
; GCN-DAZ-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -75,21 +74,22 @@ define amdgpu_kernel void @rsq_f32_sgpr(ptr addrspace(1) noalias %out, float %va
;
; GCN-IEEE-LABEL: rsq_f32_sgpr:
; GCN-IEEE: ; %bb.0:
-; GCN-IEEE-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-IEEE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; GCN-IEEE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-IEEE-NEXT: v_mov_b32_e32 v0, 0x800000
+; GCN-IEEE-NEXT: s_mov_b32 s7, 0xf000
+; GCN-IEEE-NEXT: s_mov_b32 s6, -1
; GCN-IEEE-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-IEEE-NEXT: v_cmp_lt_f32_e32 vcc, s6, v0
-; GCN-IEEE-NEXT: s_and_b64 s[2:3], vcc, exec
-; GCN-IEEE-NEXT: s_cselect_b32 s2, 24, 0
-; GCN-IEEE-NEXT: v_mov_b32_e32 v0, s2
-; GCN-IEEE-NEXT: v_ldexp_f32_e32 v0, s6, v0
+; GCN-IEEE-NEXT: v_cmp_lt_f32_e32 vcc, s2, v0
+; GCN-IEEE-NEXT: s_and_b64 s[4:5], vcc, exec
+; GCN-IEEE-NEXT: s_cselect_b32 s3, 24, 0
+; GCN-IEEE-NEXT: v_mov_b32_e32 v0, s3
+; GCN-IEEE-NEXT: v_ldexp_f32_e32 v0, s2, v0
; GCN-IEEE-NEXT: v_rsq_f32_e32 v0, v0
-; GCN-IEEE-NEXT: s_cselect_b32 s4, 12, 0
-; GCN-IEEE-NEXT: s_mov_b32 s3, 0xf000
-; GCN-IEEE-NEXT: s_mov_b32 s2, -1
-; GCN-IEEE-NEXT: v_ldexp_f32_e64 v0, v0, s4
-; GCN-IEEE-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-IEEE-NEXT: s_mov_b32 s4, s0
+; GCN-IEEE-NEXT: s_cselect_b32 s0, 12, 0
+; GCN-IEEE-NEXT: s_mov_b32 s5, s1
+; GCN-IEEE-NEXT: v_ldexp_f32_e64 v0, v0, s0
+; GCN-IEEE-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-IEEE-NEXT: s_endpgm
%sqrt = call contract float @llvm.sqrt.f32(float %val) nounwind readnone
%div = fdiv contract float 1.0, %sqrt, !fpmath !0
diff --git a/llvm/test/CodeGen/AMDGPU/rsq.f32.ll b/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
index e841cb1518532..435a4e54ca315 100644
--- a/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
+++ b/llvm/test/CodeGen/AMDGPU/rsq.f32.ll
@@ -74,10 +74,9 @@ define amdgpu_kernel void @rsq_f32(ptr addrspace(1) noalias %out, ptr addrspace(
define amdgpu_kernel void @rsq_f32_sgpr(ptr addrspace(1) noalias %out, float %val) {
; GCN-DAZ-LABEL: rsq_f32_sgpr:
; GCN-DAZ: ; %bb.0:
-; GCN-DAZ-NEXT: s_load_dword s2, s[4:5], 0xb
-; GCN-DAZ-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-DAZ-NEXT: s_mov_b32 s3, 0xf000
+; GCN-DAZ-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-DAZ-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-DAZ-NEXT: s_mov_b32 s3, 0xf000
; GCN-DAZ-NEXT: v_rsq_f32_e32 v0, s2
; GCN-DAZ-NEXT: s_mov_b32 s2, -1
; GCN-DAZ-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -85,10 +84,9 @@ define amdgpu_kernel void @rsq_f32_sgpr(ptr addrspace(1) noalias %out, float %va
;
; GCN-IEEE-LABEL: rsq_f32_sgpr:
; GCN-IEEE: ; %bb.0:
-; GCN-IEEE-NEXT: s_load_dword s2, s[4:5], 0xb
-; GCN-IEEE-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-IEEE-NEXT: s_mov_b32 s3, 0xf000
+; GCN-IEEE-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-IEEE-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-IEEE-NEXT: s_mov_b32 s3, 0xf000
; GCN-IEEE-NEXT: v_rsq_f32_e32 v0, s2
; GCN-IEEE-NEXT: s_mov_b32 s2, -1
; GCN-IEEE-NEXT: buffer_store_dword v0, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll b/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll
index f14a5cc19774d..7f267ac0c9739 100644
--- a/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/s_addk_i32.ll
@@ -5,8 +5,8 @@
; allocation differences.
; SI-LABEL: {{^}}s_addk_i32_k0:
-; SI: s_load_dword [[VAL:s[0-9]+]]
-; SI: s_addk_i32 [[VAL]], 0x41
+; SI-DAG: s_load_dwordx4
+; SI: s_add_i32 [[VAL:s[0-9]+]], s{{[0-9]+}}, 0x41
; SI: v_mov_b32_e32 [[VRESULT:v[0-9]+]], [[VAL]]
; SI: buffer_store_dword [[VRESULT]]
; SI: s_endpgm
@@ -29,7 +29,7 @@ define amdgpu_kernel void @s_addk_i32_k0_x2(ptr addrspace(1) %out0, ptr addrspac
}
; SI-LABEL: {{^}}s_addk_i32_k1:
-; SI: s_addk_i32 {{s[0-9]+}}, 0x7fff{{$}}
+; SI: s_add_i32 {{s[0-9]+}}, s{{[0-9]+}}, 0x7fff{{$}}
; SI: s_endpgm
define amdgpu_kernel void @s_addk_i32_k1(ptr addrspace(1) %out, i32 %b) #0 {
%add = add i32 %b, 32767 ; (1 << 15) - 1
@@ -47,7 +47,7 @@ define amdgpu_kernel void @s_addk_i32_k2(ptr addrspace(1) %out, i32 %b) #0 {
}
; SI-LABEL: {{^}}s_addk_i32_k3:
-; SI: s_addk_i32 {{s[0-9]+}}, 0xffbf{{$}}
+; SI: s_add_i32 {{s[0-9]+}}, s{{[0-9]+}}, 0xffffffbf{{$}}
; SI: s_endpgm
define amdgpu_kernel void @s_addk_i32_k3(ptr addrspace(1) %out, i32 %b) #0 {
%add = add i32 %b, -65
diff --git a/llvm/test/CodeGen/AMDGPU/s_mulk_i32.ll b/llvm/test/CodeGen/AMDGPU/s_mulk_i32.ll
index ab98e8125e801..86b1741359000 100644
--- a/llvm/test/CodeGen/AMDGPU/s_mulk_i32.ll
+++ b/llvm/test/CodeGen/AMDGPU/s_mulk_i32.ll
@@ -5,26 +5,28 @@
define amdgpu_kernel void @s_mulk_i32_k0(ptr addrspace(1) %out, i32 %b) {
; GFX6-LABEL: s_mulk_i32_k0:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x2
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s4, s6, 0x41
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mul_i32 s0, s2, 0x41
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: s_mulk_i32_k0:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s4, s6, 0x41
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_mul_i32 s0, s2, 0x41
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
%mul = mul i32 %b, 65
store i32 %mul, ptr addrspace(1) %out
@@ -34,26 +36,28 @@ define amdgpu_kernel void @s_mulk_i32_k0(ptr addrspace(1) %out, i32 %b) {
define amdgpu_kernel void @s_mulk_i32_k1(ptr addrspace(1) %out, i32 %b) {
; GFX6-LABEL: s_mulk_i32_k1:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x2
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s4, s6, 0x7fff
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mul_i32 s0, s2, 0x7fff
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: s_mulk_i32_k1:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s4, s6, 0x7fff
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_mul_i32 s0, s2, 0x7fff
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
%mul = mul i32 %b, 32767 ; (1 << 15) - 1
store i32 %mul, ptr addrspace(1) %out
@@ -63,26 +67,28 @@ define amdgpu_kernel void @s_mulk_i32_k1(ptr addrspace(1) %out, i32 %b) {
define amdgpu_kernel void @s_mulk_i32_k2(ptr addrspace(1) %out, i32 %b) {
; GFX6-LABEL: s_mulk_i32_k2:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x2
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s4, s6, 0xffffffef
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mul_i32 s0, s2, 0xffffffef
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: s_mulk_i32_k2:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s4, s6, 0xffffffef
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_mul_i32 s0, s2, 0xffffffef
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
%mul = mul i32 %b, -17
store i32 %mul, ptr addrspace(1) %out
@@ -92,26 +98,28 @@ define amdgpu_kernel void @s_mulk_i32_k2(ptr addrspace(1) %out, i32 %b) {
define amdgpu_kernel void @no_s_mulk_i32_k0(ptr addrspace(1) %out, i32 %b) {
; GFX6-LABEL: no_s_mulk_i32_k0:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0x2
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mul_i32 s4, s6, 0x8001
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_mul_i32 s0, s2, 0x8001
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: no_s_mulk_i32_k0:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s6, s[4:5], 0x8
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX8-NEXT: s_mov_b32 s3, 0xf000
-; GFX8-NEXT: s_mov_b32 s2, -1
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
+; GFX8-NEXT: s_mov_b32 s7, 0xf000
+; GFX8-NEXT: s_mov_b32 s6, -1
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_mul_i32 s4, s6, 0x8001
-; GFX8-NEXT: v_mov_b32_e32 v0, s4
-; GFX8-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX8-NEXT: s_mov_b32 s4, s0
+; GFX8-NEXT: s_mul_i32 s0, s2, 0x8001
+; GFX8-NEXT: s_mov_b32 s5, s1
+; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX8-NEXT: s_endpgm
%mul = mul i32 %b, 32769 ; 1 << 15 + 1
store i32 %mul, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/sad.ll b/llvm/test/CodeGen/AMDGPU/sad.ll
index 15fc987d1e7c6..ad7f685f004a0 100644
--- a/llvm/test/CodeGen/AMDGPU/sad.ll
+++ b/llvm/test/CodeGen/AMDGPU/sad.ll
@@ -4,17 +4,17 @@
define amdgpu_kernel void @v_sad_u32_pat1(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
; GCN-LABEL: v_sad_u32_pat1:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_load_dword s4, s[8:9], 0x4
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_sad_u32 v2, s0, v0, v1
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, s3
+; GCN-NEXT: v_mov_b32_e32 v3, s4
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_sad_u32 v2, s2, v2, v3
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
@@ -33,16 +33,15 @@ define amdgpu_kernel void @v_sad_u32_pat1(ptr addrspace(1) %out, i32 %a, i32 %b,
define amdgpu_kernel void @v_sad_u32_constant_pat1(ptr addrspace(1) %out, i32 %a) {
; GCN-LABEL: v_sad_u32_constant_pat1:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; GCN-NEXT: v_mov_b32_e32 v0, 0x5a
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: v_mov_b32_e32 v2, 0x5a
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_sad_u32 v2, s2, v0, 20
; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_sad_u32 v2, s2, v2, 20
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, 90
@@ -61,17 +60,17 @@ define amdgpu_kernel void @v_sad_u32_constant_pat1(ptr addrspace(1) %out, i32 %a
define amdgpu_kernel void @v_sad_u32_pat2(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) {
; GCN-LABEL: v_sad_u32_pat2:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_load_dword s4, s[8:9], 0x4
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: v_mov_b32_e32 v3, s4
; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_sad_u32 v2, s1, v0, v1
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_sad_u32 v2, s3, v2, v3
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
@@ -90,21 +89,21 @@ define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat1(ptr addrspace(1) %out, i
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[22:23], s[2:3]
; GCN-NEXT: s_mov_b64 s[20:21], s[0:1]
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_load_dword s4, s[8:9], 0x4
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_add_u32 s20, s20, s17
; GCN-NEXT: s_addc_u32 s21, s21, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_min_u32 s3, s0, s1
-; GCN-NEXT: s_max_u32 s0, s0, s1
-; GCN-NEXT: s_sub_i32 s0, s0, s3
-; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_min_u32 s0, s2, s3
+; GCN-NEXT: s_max_u32 s1, s2, s3
+; GCN-NEXT: s_sub_i32 s0, s1, s0
; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: s_add_i32 s0, s0, s2
+; GCN-NEXT: s_add_i32 s0, s0, s4
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: buffer_store_dword v2, v0, s[20:23], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v2, s0
@@ -129,19 +128,19 @@ define amdgpu_kernel void @v_sad_u32_multi_use_add_pat1(ptr addrspace(1) %out, i
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[22:23], s[2:3]
; GCN-NEXT: s_mov_b64 s[20:21], s[0:1]
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_load_dword s4, s[8:9], 0x4
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_add_u32 s20, s20, s17
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v2, s1
-; GCN-NEXT: v_mov_b32_e32 v3, s2
-; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v2, s3
+; GCN-NEXT: v_mov_b32_e32 v3, s4
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT: s_addc_u32 s21, s21, 0
-; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: v_sad_u32 v2, s0, v2, v3
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_sad_u32 v2, s2, v2, v3
; GCN-NEXT: buffer_store_dword v2, v0, s[20:23], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: flat_store_dword v[0:1], v2
@@ -164,24 +163,24 @@ define amdgpu_kernel void @v_sad_u32_multi_use_max_pat1(ptr addrspace(1) %out, i
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[22:23], s[2:3]
; GCN-NEXT: s_mov_b64 s[20:21], s[0:1]
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_load_dword s4, s[8:9], 0x4
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_add_u32 s20, s20, s17
; GCN-NEXT: s_addc_u32 s21, s21, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_max_u32 s3, s0, s1
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_max_u32 s0, s2, s3
+; GCN-NEXT: v_mov_b32_e32 v3, s0
; GCN-NEXT: v_mov_b32_e32 v2, s3
-; GCN-NEXT: v_sad_u32 v3, s0, v0, v1
-; GCN-NEXT: buffer_store_dword v2, v0, s[20:23], 0 offen
+; GCN-NEXT: buffer_store_dword v3, v0, s[20:23], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v3, s4
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: flat_store_dword v[0:1], v3
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_sad_u32 v2, s2, v2, v3
+; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%t0 = select i1 %icmp0, i32 %a, i32 %b
@@ -202,24 +201,24 @@ define amdgpu_kernel void @v_sad_u32_multi_use_min_pat1(ptr addrspace(1) %out, i
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[22:23], s[2:3]
; GCN-NEXT: s_mov_b64 s[20:21], s[0:1]
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_load_dword s4, s[8:9], 0x4
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_add_u32 s20, s20, s17
; GCN-NEXT: s_addc_u32 s21, s21, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_min_u32 s3, s0, s1
-; GCN-NEXT: v_mov_b32_e32 v0, s1
-; GCN-NEXT: v_mov_b32_e32 v1, s2
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_min_u32 s0, s2, s3
+; GCN-NEXT: v_mov_b32_e32 v3, s0
; GCN-NEXT: v_mov_b32_e32 v2, s3
-; GCN-NEXT: v_sad_u32 v3, s0, v0, v1
-; GCN-NEXT: buffer_store_dword v2, v0, s[20:23], 0 offen
+; GCN-NEXT: buffer_store_dword v3, v0, s[20:23], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v3, s4
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: flat_store_dword v[0:1], v3
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_sad_u32 v2, s2, v2, v3
+; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%t0 = select i1 %icmp0, i32 %a, i32 %b
@@ -241,24 +240,24 @@ define amdgpu_kernel void @v_sad_u32_multi_use_sub_pat2(ptr addrspace(1) %out, i
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[22:23], s[2:3]
; GCN-NEXT: s_mov_b64 s[20:21], s[0:1]
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_load_dword s4, s[8:9], 0x4
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_add_u32 s20, s20, s17
; GCN-NEXT: s_addc_u32 s21, s21, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_sub_i32 s3, s0, s1
; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_mov_b32_e32 v2, s3
-; GCN-NEXT: v_sad_u32 v3, s1, v0, v1
-; GCN-NEXT: buffer_store_dword v2, v0, s[20:23], 0 offen
+; GCN-NEXT: s_sub_i32 s0, s2, s3
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: buffer_store_dword v3, v0, s[20:23], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v3, s4
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: flat_store_dword v[0:1], v3
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_sad_u32 v2, s3, v2, v3
+; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
%sub0 = sub i32 %a, %b
@@ -277,21 +276,21 @@ define amdgpu_kernel void @v_sad_u32_multi_use_select_pat2(ptr addrspace(1) %out
; GCN: ; %bb.0:
; GCN-NEXT: s_mov_b64 s[22:23], s[2:3]
; GCN-NEXT: s_mov_b64 s[20:21], s[0:1]
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_load_dword s4, s[8:9], 0x4
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_add_u32 s20, s20, s17
; GCN-NEXT: s_addc_u32 s21, s21, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_min_u32 s3, s1, s0
-; GCN-NEXT: s_max_u32 s0, s1, s0
-; GCN-NEXT: s_sub_i32 s0, s0, s3
-; GCN-NEXT: v_mov_b32_e32 v0, s4
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_min_u32 s0, s3, s2
+; GCN-NEXT: s_max_u32 s1, s3, s2
+; GCN-NEXT: s_sub_i32 s0, s1, s0
; GCN-NEXT: v_mov_b32_e32 v2, s0
-; GCN-NEXT: s_add_i32 s0, s0, s2
+; GCN-NEXT: s_add_i32 s0, s0, s4
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
-; GCN-NEXT: v_mov_b32_e32 v1, s5
; GCN-NEXT: buffer_store_dword v2, v0, s[20:23], 0 offen
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v2, s0
@@ -393,13 +392,13 @@ define amdgpu_kernel void @v_sad_u32_i16_pat1(ptr addrspace(1) %out, i16 %a, i16
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_and_b32 s4, s2, 0xffff
-; GCN-NEXT: s_lshr_b32 s2, s2, 16
-; GCN-NEXT: v_mov_b32_e32 v0, s3
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_sad_u32 v2, s4, v1, v0
-; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_lshr_b32 s1, s2, 16
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_and_b32 s0, s2, 0xffff
+; GCN-NEXT: v_mov_b32_e32 v2, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s3
+; GCN-NEXT: v_sad_u32 v2, s0, v2, v3
; GCN-NEXT: flat_store_short v[0:1], v2
; GCN-NEXT: s_endpgm
%icmp0 = icmp ugt i16 %a, %b
@@ -451,20 +450,19 @@ define amdgpu_kernel void @v_sad_u32_i16_pat2(ptr addrspace(1) %out) {
define amdgpu_kernel void @v_sad_u32_i8_pat1(ptr addrspace(1) %out, i8 %a, i8 %b, i8 %c) {
; GCN-LABEL: v_sad_u32_i8_pat1:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_and_b32 s3, s2, 0xff
-; GCN-NEXT: s_bfe_u32 s4, s2, 0x80008
-; GCN-NEXT: s_lshr_b32 s2, s2, 16
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_sad_u32 v2, s3, v0, v1
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_and_b32 s0, s2, 0xff
+; GCN-NEXT: s_bfe_u32 s1, s2, 0x80008
+; GCN-NEXT: s_lshr_b32 s2, s2, 16
+; GCN-NEXT: v_mov_b32_e32 v2, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s2
+; GCN-NEXT: v_sad_u32 v2, s0, v2, v3
; GCN-NEXT: flat_store_byte v[0:1], v2
; GCN-NEXT: s_endpgm
%icmp0 = icmp ugt i8 %a, %b
@@ -516,20 +514,19 @@ define amdgpu_kernel void @v_sad_u32_i8_pat2(ptr addrspace(1) %out) {
define amdgpu_kernel void @s_sad_u32_i8_pat2(ptr addrspace(1) %out, i8 zeroext %a, i8 zeroext %b, i8 zeroext %c) {
; GCN-LABEL: s_sad_u32_i8_pat2:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s2, s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_and_b32 s3, s2, 0xff
-; GCN-NEXT: s_bfe_u32 s4, s2, 0x80008
-; GCN-NEXT: s_lshr_b32 s2, s2, 16
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s2
-; GCN-NEXT: v_sad_u32 v2, s3, v0, v1
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_and_b32 s0, s2, 0xff
+; GCN-NEXT: s_bfe_u32 s1, s2, 0x80008
+; GCN-NEXT: s_lshr_b32 s2, s2, 16
+; GCN-NEXT: v_mov_b32_e32 v2, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s2
+; GCN-NEXT: v_sad_u32 v2, s0, v2, v3
; GCN-NEXT: flat_store_byte v[0:1], v2
; GCN-NEXT: s_endpgm
%icmp0 = icmp ugt i8 %a, %b
@@ -546,19 +543,19 @@ define amdgpu_kernel void @s_sad_u32_i8_pat2(ptr addrspace(1) %out, i8 zeroext %
define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat1(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c, i32 %d) {
; GCN-LABEL: v_sad_u32_mismatched_operands_pat1:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x4
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_max_u32 s6, s0, s1
-; GCN-NEXT: s_cmp_le_u32 s0, s1
-; GCN-NEXT: s_cselect_b32 s0, s0, s3
-; GCN-NEXT: s_sub_i32 s0, s6, s0
-; GCN-NEXT: s_add_i32 s0, s0, s2
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_max_u32 s0, s2, s3
+; GCN-NEXT: s_cmp_le_u32 s2, s3
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: s_cselect_b32 s1, s2, s5
+; GCN-NEXT: s_sub_i32 s0, s0, s1
+; GCN-NEXT: s_add_i32 s0, s0, s4
; GCN-NEXT: v_mov_b32_e32 v2, s0
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
@@ -578,20 +575,20 @@ define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat1(ptr addrspace(1) %
define amdgpu_kernel void @v_sad_u32_mismatched_operands_pat2(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c, i32 %d) {
; GCN-LABEL: v_sad_u32_mismatched_operands_pat2:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x2
-; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x4
; GCN-NEXT: s_add_i32 s12, s12, s17
; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: s_sub_i32 s0, s0, s3
-; GCN-NEXT: v_mov_b32_e32 v1, s0
-; GCN-NEXT: v_sub_i32_e32 v0, vcc, s1, v0
-; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-NEXT: v_add_i32_e32 v2, vcc, s2, v0
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s5
+; GCN-NEXT: v_mov_b32_e32 v2, s2
+; GCN-NEXT: s_sub_i32 s0, s2, s5
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: v_sub_i32_e32 v2, vcc, s3, v2
+; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_add_i32_e32 v2, vcc, s4, v2
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
%icmp0 = icmp ugt i32 %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
index a82415e2e8303..54687326cef2e 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
@@ -393,33 +393,33 @@ bb:
define amdgpu_kernel void @scalar_to_vector_test6(ptr addrspace(1) %out, i8 zeroext %val) nounwind {
; SI-LABEL: scalar_to_vector_test6:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; GFX89-LABEL: scalar_to_vector_test6:
; GFX89: ; %bb.0:
-; GFX89-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX89-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX89-NEXT: s_mov_b32 s3, 0xf000
-; GFX89-NEXT: s_mov_b32 s2, -1
+; GFX89-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX89-NEXT: s_mov_b32 s7, 0xf000
+; GFX89-NEXT: s_mov_b32 s6, -1
; GFX89-NEXT: s_waitcnt lgkmcnt(0)
-; GFX89-NEXT: v_mov_b32_e32 v0, s6
-; GFX89-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX89-NEXT: s_mov_b32 s4, s0
+; GFX89-NEXT: s_mov_b32 s5, s1
+; GFX89-NEXT: v_mov_b32_e32 v0, s2
+; GFX89-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX89-NEXT: s_endpgm
;
; GFX11-LABEL: scalar_to_vector_test6:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX11-NEXT: s_mov_b32 s3, 0x31016000
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-NEXT: v_mov_b32_e32 v0, s2
; GFX11-NEXT: s_mov_b32 s2, -1
; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-commute.ll b/llvm/test/CodeGen/AMDGPU/sdwa-commute.ll
index 8ad6a4e534d23..dbe2b327be91f 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-commute.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-commute.ll
@@ -19,7 +19,7 @@ define void @extracted_values(ptr %ret_struct, ptr addrspace(3) %arg0, ptr addrs
; CHECK-NEXT: v_sub_f16_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; CHECK-NEXT: v_add_f16_e32 v4, v6, v7
; CHECK-NEXT: v_add_f16_e32 v2, v3, v2
-; CHECK-NEXT: v_pack_b32_f16 v2, v4, v2
+; CHECK-NEXT: v_perm_b32 v2, v2, v4, 0x5040100
; CHECK-NEXT: flat_store_dword v[0:1], v2
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/sdwa-peephole-instr-combine-sel.ll b/llvm/test/CodeGen/AMDGPU/sdwa-peephole-instr-combine-sel.ll
index 7ac215eee9c92..c0d6d021cfcae 100644
--- a/llvm/test/CodeGen/AMDGPU/sdwa-peephole-instr-combine-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/sdwa-peephole-instr-combine-sel.ll
@@ -10,9 +10,7 @@
define amdgpu_kernel void @widget(ptr addrspace(1) %arg, i1 %arg1, ptr addrspace(3) %arg2, ptr addrspace(3) %arg3) {
; CHECK-LABEL: widget:
; CHECK: ; %bb.0: ; %bb
-; CHECK-NEXT: s_clause 0x1
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CHECK-NEXT: s_load_dword s2, s[8:9], 0x8
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: global_load_sbyte v0, v0, s[0:1] offset:2
; CHECK-NEXT: s_bitcmp1_b32 s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/select-vectors.ll b/llvm/test/CodeGen/AMDGPU/select-vectors.ll
index e754f665c5f43..e6e9f96b00bde 100644
--- a/llvm/test/CodeGen/AMDGPU/select-vectors.ll
+++ b/llvm/test/CodeGen/AMDGPU/select-vectors.ll
@@ -360,8 +360,6 @@ define amdgpu_kernel void @select_v8f32(ptr addrspace(1) %out, <8 x float> %a, <
; GCN-LABEL: {{^}}select_v2f64:
; GCN: s_cselect_b32
; GCN: s_cselect_b32
-; GCN: s_cselect_b32
-; GCN: s_cselect_b32
define amdgpu_kernel void @select_v2f64(ptr addrspace(1) %out, <2 x double> %a, <2 x double> %b, i32 %c) #0 {
%cmp = icmp eq i32 %c, 0
%select = select i1 %cmp, <2 x double> %a, <2 x double> %b
diff --git a/llvm/test/CodeGen/AMDGPU/setcc-opt.ll b/llvm/test/CodeGen/AMDGPU/setcc-opt.ll
index 031a55a2dc758..5fb84c0598b5b 100644
--- a/llvm/test/CodeGen/AMDGPU/setcc-opt.ll
+++ b/llvm/test/CodeGen/AMDGPU/setcc-opt.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=SI -check-prefix=GCN -check-prefix=FUNC %s
-; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=VI -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
+; RUN: llc -mtriple=amdgcn -mcpu=tonga -mattr=-flat-for-global < %s | FileCheck -check-prefix=GCN -check-prefix=FUNC %s
; RUN: llc -mtriple=r600 -mcpu=cypress < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
; FUNC-LABEL: {{^}}sext_bool_icmp_eq_0:
@@ -154,8 +154,8 @@ define amdgpu_kernel void @zext_bool_icmp_ne_neg1(ptr addrspace(1) %out, i32 %a,
}
; FUNC-LABEL: {{^}}cmp_zext_k_i8max:
-; GCN: s_load_dword [[VALUE:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}
-; GCN-DAG: s_and_b32 [[B:s[0-9]+]], [[VALUE]], 0xff
+; GCN-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x9|0x24}}
+; GCN-DAG: s_and_b32 [[B:s[0-9]+]], s{{[0-9]+}}, 0xff
; GCN: s_cmpk_lg_i32 [[B]], 0xff
; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
@@ -199,9 +199,8 @@ define void @v_cmp_sext_k_neg1_i8_sext_arg(i8 signext %b) nounwind {
; Should do a buffer_load_sbyte and compare with -1
; FUNC-LABEL: {{^}}cmp_sext_k_neg1_i8_arg:
-; SI: s_load_dword [[VAL:s[0-9]+]], s[{{[0-9]+:[0-9]+}}], 0xb
-; VI: s_load_dword [[VAL:s[0-9]+]], s{{\[[0-9]+:[0-9]+\]}}, 0x2c
-; GCN-DAG: s_and_b32 [[B:s[0-9]+]], [[VAL]], 0xff
+; GCN-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x9|0x24}}
+; GCN-DAG: s_and_b32 [[B:s[0-9]+]], s{{[0-9]+}}, 0xff
; GCN: s_cmpk_lg_i32 [[B]], 0xff{{$}}
; GCN: s_cselect_b64 [[CC:[^,]+]], -1, 0
; GCN: v_cndmask_b32_e64 [[RESULT:v[0-9]+]], 0, 1, [[CC]]
diff --git a/llvm/test/CodeGen/AMDGPU/sext-divergence-driven-isel.ll b/llvm/test/CodeGen/AMDGPU/sext-divergence-driven-isel.ll
index e589a6341ea0e..ad7694614050c 100644
--- a/llvm/test/CodeGen/AMDGPU/sext-divergence-driven-isel.ll
+++ b/llvm/test/CodeGen/AMDGPU/sext-divergence-driven-isel.ll
@@ -5,14 +5,15 @@ define amdgpu_kernel void @sext_i16_to_i32_uniform(ptr addrspace(1) %out, i16 %a
; GCN-LABEL: sext_i16_to_i32_uniform:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_sext_i32_i16 s4, s4
-; GCN-NEXT: s_add_i32 s4, s5, s4
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_sext_i32_i16 s0, s2
+; GCN-NEXT: s_add_i32 s0, s3, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%sext = sext i16 %a to i32
%res = add i32 %b, %sext
@@ -24,18 +25,19 @@ define amdgpu_kernel void @sext_i16_to_i32_uniform(ptr addrspace(1) %out, i16 %a
define amdgpu_kernel void @sext_i16_to_i64_uniform(ptr addrspace(1) %out, i16 %a, i64 %b) {
; GCN-LABEL: sext_i16_to_i64_uniform:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GCN-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_bfe_i64 s[4:5], s[6:7], 0x100000
-; GCN-NEXT: s_add_u32 s4, s8, s4
-; GCN-NEXT: s_addc_u32 s5, s9, s5
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_bfe_i64 s[0:1], s[2:3], 0x100000
+; GCN-NEXT: s_add_u32 s0, s8, s0
+; GCN-NEXT: s_addc_u32 s1, s9, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-NEXT: s_endpgm
%sext = sext i16 %a to i64
%res = add i64 %b, %sext
@@ -46,14 +48,15 @@ define amdgpu_kernel void @sext_i16_to_i64_uniform(ptr addrspace(1) %out, i16 %a
define amdgpu_kernel void @sext_i16_to_i32_divergent(ptr addrspace(1) %out, i16 %a, i32 %b) {
; GCN-LABEL: sext_i16_to_i32_divergent:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
; GCN-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.truncated = trunc i32 %tid to i16
@@ -67,15 +70,16 @@ define amdgpu_kernel void @sext_i16_to_i32_divergent(ptr addrspace(1) %out, i16
define amdgpu_kernel void @sext_i16_to_i64_divergent(ptr addrspace(1) %out, i16 %a, i64 %b) {
; GCN-LABEL: sext_i16_to_i64_divergent:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s2, v0
; GCN-NEXT: v_bfe_i32 v0, v0, 0, 16
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.truncated = trunc i32 %tid to i16
@@ -88,18 +92,19 @@ define amdgpu_kernel void @sext_i16_to_i64_divergent(ptr addrspace(1) %out, i16
define amdgpu_kernel void @sext_i32_to_i64_uniform(ptr addrspace(1) %out, i32 %a, i64 %b) {
; GCN-LABEL: sext_i32_to_i64_uniform:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s8, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_ashr_i32 s5, s8, 31
-; GCN-NEXT: s_add_u32 s4, s6, s8
-; GCN-NEXT: s_addc_u32 s5, s7, s5
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_ashr_i32 s1, s2, 31
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_add_u32 s0, s8, s2
+; GCN-NEXT: s_addc_u32 s1, s9, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-NEXT: s_endpgm
%sext = sext i32 %a to i64
%res = add i64 %b, %sext
@@ -110,14 +115,15 @@ define amdgpu_kernel void @sext_i32_to_i64_uniform(ptr addrspace(1) %out, i32 %a
define amdgpu_kernel void @sext_i32_to_i64_divergent(ptr addrspace(1) %out, i32 %a, i64 %b) {
; GCN-LABEL: sext_i32_to_i64_divergent:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
; GCN-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%divergent.a = add i32 %a, %tid
diff --git a/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll b/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll
index 65fa2ca4dbe9c..cdb1392ba83ca 100644
--- a/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll
+++ b/llvm/test/CodeGen/AMDGPU/sext-in-reg.ll
@@ -7,8 +7,8 @@
; FIXME: r600 fails verifier
; FUNC-LABEL: {{^}}sext_in_reg_i1_i32:
-; GCN: s_load_dword [[ARG:s[0-9]+]],
-; GCN: s_bfe_i32 [[SEXTRACT:s[0-9]+]], [[ARG]], 0x10000
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN: s_bfe_i32 [[SEXTRACT:s[0-9]+]], s{{[0-9]+}}, 0x10000
; GCN: v_mov_b32_e32 [[EXTRACT:v[0-9]+]], [[SEXTRACT]]
; GCN: buffer_store_dword [[EXTRACT]],
@@ -519,6 +519,7 @@ define amdgpu_kernel void @v_sext_in_reg_i32_to_i64_move_use(ptr addrspace(1) %o
}
; FUNC-LABEL: {{^}}s_sext_in_reg_i1_i16:
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
; GCN: s_load_dword [[VAL:s[0-9]+]]
; GCN: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0x10000
@@ -535,6 +536,7 @@ define amdgpu_kernel void @s_sext_in_reg_i1_i16(ptr addrspace(1) %out, ptr addrs
}
; FUNC-LABEL: {{^}}s_sext_in_reg_i2_i16:
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
; GCN: s_load_dword [[VAL:s[0-9]+]]
; SI: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0x20000
@@ -596,9 +598,9 @@ define amdgpu_kernel void @v_sext_in_reg_i1_i16_nonload(ptr addrspace(3) %out, p
}
; FUNC-LABEL: {{^}}s_sext_in_reg_i2_i16_arg:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
-; SI: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0x20000
+; SI: s_bfe_i32 [[BFE:s[0-9]+]], s{{[0-9]+}}, 0x20000
; SI: v_mov_b32_e32 [[VBFE:v[0-9]+]], [[BFE]]
; SI: buffer_store_short [[VBFE]]
@@ -613,9 +615,9 @@ define amdgpu_kernel void @s_sext_in_reg_i2_i16_arg(ptr addrspace(1) %out, i16 %
}
; FUNC-LABEL: {{^}}s_sext_in_reg_i8_i16_arg:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
-; SI: s_sext_i32_i8 [[SSEXT:s[0-9]+]], [[VAL]]
+; SI: s_sext_i32_i8 [[SSEXT:s[0-9]+]], s{{[0-9]+}}
; SI: v_mov_b32_e32 [[VSEXT:v[0-9]+]], [[SSEXT]]
; SI: buffer_store_short [[VSEXT]]
@@ -628,9 +630,9 @@ define amdgpu_kernel void @s_sext_in_reg_i8_i16_arg(ptr addrspace(1) %out, i16 %
}
; FUNC-LABEL: {{^}}s_sext_in_reg_i15_i16_arg:
-; GCN: s_load_dword [[VAL:s[0-9]+]]
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
-; SI: s_bfe_i32 [[BFE:s[0-9]+]], [[VAL]], 0xf0000
+; SI: s_bfe_i32 [[BFE:s[0-9]+]], s{{[0-9]+}}, 0xf0000
; SI: v_mov_b32_e32 [[VBFE:v[0-9]+]], [[BFE]]
; SI: buffer_store_short [[VBFE]]
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll b/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
index 40b6f0262f979..9194b1b36aee3 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-control-flow.ll
@@ -11,27 +11,26 @@
define amdgpu_kernel void @sgpr_if_else_salu_br(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) {
; SI-LABEL: sgpr_if_else_salu_br:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dword s6, s[4:5], 0xf
+; SI-NEXT: s_load_dwordx8 s[0:7], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cmp_lg_u32 s2, 0
; SI-NEXT: s_cbranch_scc0 .LBB0_4
; SI-NEXT: ; %bb.1: ; %else
-; SI-NEXT: s_add_i32 s3, s3, s6
+; SI-NEXT: s_add_i32 s5, s5, s6
; SI-NEXT: s_cbranch_execnz .LBB0_3
; SI-NEXT: .LBB0_2: ; %if
-; SI-NEXT: s_sub_i32 s3, s1, s2
+; SI-NEXT: s_sub_i32 s5, s3, s4
; SI-NEXT: .LBB0_3: ; %endif
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-NEXT: s_add_i32 s0, s3, s0
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_add_i32 s0, s5, s2
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s9, s1
; SI-NEXT: v_mov_b32_e32 v0, s0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SI-NEXT: s_endpgm
; SI-NEXT: .LBB0_4:
-; SI-NEXT: ; implicit-def: $sgpr3
+; SI-NEXT: ; implicit-def: $sgpr5
; SI-NEXT: s_branch .LBB0_2
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-copy-local-cse.ll b/llvm/test/CodeGen/AMDGPU/sgpr-copy-local-cse.ll
index c3a19117385e1..37699bdd18c5a 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-copy-local-cse.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-copy-local-cse.ll
@@ -1,10 +1,10 @@
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx906 -o - %s | FileCheck %s
; CHECK-LABEL: {{^}}t0:
-; CHECK: s_load_dwordx2 s[[[PTR_LO:[0-9]+]]:[[PTR_HI:[0-9]+]]], s[8:9], 0x0
-; CHECK: v_mov_b32_e32 v{{[0-9]+}}, s[[PTR_HI]]
-; There should be no redundant copies from PTR_HI.
-; CHECK-NOT: v_mov_b32_e32 v{{[0-9]+}}, s[[PTR_HI]]
+; CHECK: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; CHECK: s_load_dword s{{[0-9]+}}, s[8:9], 0x10
+; CHECK: s_waitcnt lgkmcnt(0)
+; CHECK: v_mov_b32_e32 v{{[0-9]+}}, [[PTR_HI:s[0-9]+]]
define protected amdgpu_kernel void @t0(ptr addrspace(1) %p, i32 %i0, i32 %j0, i32 %k0) {
entry:
%0 = tail call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/shl.ll b/llvm/test/CodeGen/AMDGPU/shl.ll
index 7278247d9b45f..112a560760c61 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.ll
@@ -472,26 +472,28 @@ define amdgpu_kernel void @shl_i16_computed_amount(ptr addrspace(1) %out, ptr ad
define amdgpu_kernel void @shl_i16_i_s(ptr addrspace(1) %out, i16 zeroext %a) {
; SI-LABEL: shl_i16_i_s:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_lshl_b32 s4, s6, 12
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_lshl_b32 s0, s2, 12
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: shl_i16_i_s:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshl_b32 s4, s6, 12
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_lshl_b32 s0, s2, 12
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; EG-LABEL: shl_i16_i_s:
diff --git a/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll b/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll
index 17db3799b0de5..d8511c8f6be11 100644
--- a/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl.v2i16.ll
@@ -22,57 +22,63 @@ define amdgpu_kernel void @s_shl_v2i16(ptr addrspace(1) %out, <2 x i16> %lhs, <2
; VI-LABEL: s_shl_v2i16:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: s_lshr_b32 s6, s5, 16
-; VI-NEXT: s_lshr_b32 s7, s4, 16
-; VI-NEXT: s_lshl_b32 s4, s4, s5
-; VI-NEXT: s_lshl_b32 s5, s7, s6
-; VI-NEXT: s_lshl_b32 s5, s5, 16
-; VI-NEXT: s_and_b32 s4, s4, 0xffff
-; VI-NEXT: s_or_b32 s4, s4, s5
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_lshr_b32 s0, s3, 16
+; VI-NEXT: s_lshr_b32 s1, s2, 16
+; VI-NEXT: s_lshl_b32 s0, s1, s0
+; VI-NEXT: s_lshl_b32 s1, s2, s3
+; VI-NEXT: s_lshl_b32 s0, s0, 16
+; VI-NEXT: s_and_b32 s1, s1, 0xffff
+; VI-NEXT: s_or_b32 s0, s1, s0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; CI-LABEL: s_shl_v2i16:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; CI-NEXT: s_lshr_b32 s6, s4, 16
-; CI-NEXT: s_lshr_b32 s7, s5, 16
-; CI-NEXT: s_lshl_b32 s4, s4, s5
-; CI-NEXT: s_lshl_b32 s5, s6, s7
-; CI-NEXT: s_lshl_b32 s5, s5, 16
-; CI-NEXT: s_and_b32 s4, s4, 0xffff
-; CI-NEXT: s_or_b32 s4, s4, s5
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, -1
-; CI-NEXT: v_mov_b32_e32 v0, s4
-; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; CI-NEXT: s_mov_b32 s4, s0
+; CI-NEXT: s_mov_b32 s5, s1
+; CI-NEXT: s_lshr_b32 s0, s2, 16
+; CI-NEXT: s_lshr_b32 s1, s3, 16
+; CI-NEXT: s_lshl_b32 s0, s0, s1
+; CI-NEXT: s_lshl_b32 s1, s2, s3
+; CI-NEXT: s_lshl_b32 s0, s0, 16
+; CI-NEXT: s_and_b32 s1, s1, 0xffff
+; CI-NEXT: s_or_b32 s0, s1, s0
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
;
; GFX10-LABEL: s_shl_v2i16:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s7, 0x31016000
+; GFX10-NEXT: s_mov_b32 s6, -1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_pk_lshlrev_b16 v0, s3, s2
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000
-; GFX10-NEXT: s_mov_b32 s2, -1
-; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX10-NEXT: s_mov_b32 s4, s0
+; GFX10-NEXT: s_mov_b32 s5, s1
+; GFX10-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_shl_v2i16:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-NEXT: s_mov_b32 s6, -1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_pk_lshlrev_b16 v0, s3, s2
-; GFX11-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-NEXT: s_mov_b32 s2, -1
-; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-NEXT: s_mov_b32 s4, s0
+; GFX11-NEXT: s_mov_b32 s5, s1
+; GFX11-NEXT: buffer_store_b32 v0, off, s[4:7], 0
; GFX11-NEXT: s_endpgm
%result = shl <2 x i16> %lhs, %rhs
store <2 x i16> %result, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll b/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll
index e8da10c32f5d4..6b616cba77df9 100644
--- a/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-annotate-cf.ll
@@ -5,51 +5,47 @@
define amdgpu_kernel void @break_inserted_outside_of_loop(ptr addrspace(1) %out, i32 %a) {
; SI-LABEL: break_inserted_outside_of_loop:
; SI: ; %bb.0: ; %main_body
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: v_mbcnt_lo_u32_b32_e64 v0, -1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_and_b32_e32 v0, s0, v0
+; SI-NEXT: v_and_b32_e32 v0, s2, v0
; SI-NEXT: v_and_b32_e32 v0, 1, v0
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: s_mov_b64 s[2:3], 0
; SI-NEXT: .LBB0_1: ; %ENDIF
; SI-NEXT: ; =>This Inner Loop Header: Depth=1
-; SI-NEXT: s_and_b64 s[2:3], exec, vcc
-; SI-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
-; SI-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; SI-NEXT: s_and_b64 s[4:5], exec, vcc
+; SI-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
+; SI-NEXT: s_andn2_b64 exec, exec, s[2:3]
; SI-NEXT: s_cbranch_execnz .LBB0_1
; SI-NEXT: ; %bb.2: ; %ENDLOOP
-; SI-NEXT: s_or_b64 exec, exec, s[0:1]
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_or_b64 exec, exec, s[2:3]
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; FLAT-LABEL: break_inserted_outside_of_loop:
; FLAT: ; %bb.0: ; %main_body
-; FLAT-NEXT: s_load_dword s0, s[4:5], 0x2c
+; FLAT-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; FLAT-NEXT: v_mbcnt_lo_u32_b32 v0, -1, 0
; FLAT-NEXT: s_waitcnt lgkmcnt(0)
-; FLAT-NEXT: v_and_b32_e32 v0, s0, v0
+; FLAT-NEXT: v_and_b32_e32 v0, s2, v0
; FLAT-NEXT: v_and_b32_e32 v0, 1, v0
; FLAT-NEXT: v_cmp_eq_u32_e32 vcc, 1, v0
-; FLAT-NEXT: s_mov_b64 s[0:1], 0
+; FLAT-NEXT: s_mov_b64 s[2:3], 0
; FLAT-NEXT: .LBB0_1: ; %ENDIF
; FLAT-NEXT: ; =>This Inner Loop Header: Depth=1
-; FLAT-NEXT: s_and_b64 s[2:3], exec, vcc
-; FLAT-NEXT: s_or_b64 s[0:1], s[2:3], s[0:1]
-; FLAT-NEXT: s_andn2_b64 exec, exec, s[0:1]
+; FLAT-NEXT: s_and_b64 s[4:5], exec, vcc
+; FLAT-NEXT: s_or_b64 s[2:3], s[4:5], s[2:3]
+; FLAT-NEXT: s_andn2_b64 exec, exec, s[2:3]
; FLAT-NEXT: s_cbranch_execnz .LBB0_1
; FLAT-NEXT: ; %bb.2: ; %ENDLOOP
-; FLAT-NEXT: s_or_b64 exec, exec, s[0:1]
-; FLAT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; FLAT-NEXT: s_or_b64 exec, exec, s[2:3]
; FLAT-NEXT: s_mov_b32 s3, 0xf000
; FLAT-NEXT: s_mov_b32 s2, -1
; FLAT-NEXT: v_mov_b32_e32 v0, 0
-; FLAT-NEXT: s_waitcnt lgkmcnt(0)
; FLAT-NEXT: buffer_store_dword v0, off, s[0:3], 0
; FLAT-NEXT: s_endpgm
main_body:
diff --git a/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll b/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
index fb336f4558a63..d2373dcd29859 100644
--- a/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-triv-disjoint-mem-access.ll
@@ -284,42 +284,42 @@ define amdgpu_kernel void @reorder_constant_load_local_store_constant_load(ptr a
; CI-NEXT: v_mov_b32_e32 v0, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: ds_read_b64 v[0:1], v0
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; CI-NEXT: s_load_dword s6, s[4:5], 0xb
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_readfirstlane_b32 s4, v0
-; CI-NEXT: v_readfirstlane_b32 s5, v1
-; CI-NEXT: s_load_dword s7, s[4:5], 0x1
-; CI-NEXT: s_load_dword s4, s[4:5], 0x3
+; CI-NEXT: v_readfirstlane_b32 s8, v0
+; CI-NEXT: v_readfirstlane_b32 s9, v1
+; CI-NEXT: s_mov_b32 s4, s0
+; CI-NEXT: s_load_dword s0, s[8:9], 0x1
+; CI-NEXT: s_load_dword s3, s[8:9], 0x3
; CI-NEXT: v_mov_b32_e32 v0, 0x63
-; CI-NEXT: v_mov_b32_e32 v1, s6
+; CI-NEXT: v_mov_b32_e32 v1, s2
+; CI-NEXT: s_mov_b32 s5, s1
; CI-NEXT: ds_write_b32 v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_add_i32 s4, s7, s4
-; CI-NEXT: v_mov_b32_e32 v0, s4
-; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; CI-NEXT: s_add_i32 s0, s0, s3
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
;
; GFX9-LABEL: reorder_constant_load_local_store_constant_load:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mov_b32_e32 v2, 0
; GFX9-NEXT: ds_read_b64 v[0:1], v2
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_readfirstlane_b32 s0, v0
-; GFX9-NEXT: v_readfirstlane_b32 s1, v1
-; GFX9-NEXT: s_load_dword s7, s[0:1], 0x4
-; GFX9-NEXT: s_load_dword s8, s[0:1], 0xc
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: v_readfirstlane_b32 s4, v0
+; GFX9-NEXT: v_readfirstlane_b32 s5, v1
+; GFX9-NEXT: s_load_dword s3, s[4:5], 0x4
+; GFX9-NEXT: s_load_dword s6, s[4:5], 0xc
; GFX9-NEXT: v_mov_b32_e32 v0, 0x63
-; GFX9-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: ds_write_b32 v1, v0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_add_i32 s0, s7, s8
-; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: global_store_dword v2, v0, s[2:3]
+; GFX9-NEXT: s_add_i32 s2, s3, s6
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: global_store_dword v2, v0, s[0:1]
; GFX9-NEXT: s_endpgm
%ptr0 = load ptr addrspace(4), ptr addrspace(3) @stored_constant_ptr, align 8
@@ -339,38 +339,38 @@ define amdgpu_kernel void @reorder_constant_load_local_store_constant_load(ptr a
define amdgpu_kernel void @reorder_smrd_load_local_store_smrd_load(ptr addrspace(1) %out, ptr addrspace(3) noalias %lptr, ptr addrspace(4) %ptr0) #0 {
; CI-LABEL: reorder_smrd_load_local_store_smrd_load:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; CI-NEXT: s_load_dword s8, s[4:5], 0xb
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; CI-NEXT: v_mov_b32_e32 v0, 0x63
; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: s_mov_b32 s7, 0xf000
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x1
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: v_mov_b32_e32 v1, s8
-; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_mov_b32 s4, s0
+; CI-NEXT: s_mov_b32 s5, s1
+; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x1
+; CI-NEXT: v_mov_b32_e32 v1, s2
+; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: ds_write_b32 v1, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_add_i32 s4, s4, s5
-; CI-NEXT: v_mov_b32_e32 v0, s4
-; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; CI-NEXT: s_add_i32 s0, s0, s1
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
;
; GFX9-LABEL: reorder_smrd_load_local_store_smrd_load:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
-; GFX9-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0x63
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x4
-; GFX9-NEXT: v_mov_b32_e32 v2, s6
+; GFX9-NEXT: s_load_dwordx2 s[4:5], s[6:7], 0x4
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: ds_write_b32 v2, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: s_add_i32 s0, s4, s5
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
-; GFX9-NEXT: global_store_dword v0, v1, s[2:3]
+; GFX9-NEXT: s_add_i32 s2, s4, s5
+; GFX9-NEXT: v_mov_b32_e32 v1, s2
+; GFX9-NEXT: global_store_dword v0, v1, s[0:1]
; GFX9-NEXT: s_endpgm
%ptr1 = getelementptr inbounds i32, ptr addrspace(4) %ptr0, i64 1
%ptr2 = getelementptr inbounds i32, ptr addrspace(4) %ptr0, i64 2
@@ -396,12 +396,13 @@ define amdgpu_kernel void @reorder_global_load_local_store_global_load(ptr addrs
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: buffer_load_dword v0, off, s[0:3], 0 offset:4
; CI-NEXT: buffer_load_dword v1, off, s[0:3], 0 offset:12
-; CI-NEXT: s_load_dword s0, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; CI-NEXT: v_mov_b32_e32 v2, 0x63
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_mov_b32_e32 v3, s0
+; CI-NEXT: s_mov_b32 s4, s0
+; CI-NEXT: s_mov_b32 s5, s1
+; CI-NEXT: v_mov_b32_e32 v3, s2
; CI-NEXT: ds_write_b32 v3, v2
; CI-NEXT: s_waitcnt vmcnt(0)
; CI-NEXT: v_add_i32_e32 v0, vcc, v0, v1
@@ -416,8 +417,7 @@ define amdgpu_kernel void @reorder_global_load_local_store_global_load(ptr addrs
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: global_load_dword v1, v0, s[0:1] offset:4
; GFX9-NEXT: global_load_dword v2, v0, s[0:1] offset:12
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v4, s2
; GFX9-NEXT: ds_write_b32 v4, v3
diff --git a/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll b/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll
index 004c27971131d..45848bdf9b5e4 100644
--- a/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll
+++ b/llvm/test/CodeGen/AMDGPU/si-unify-exit-multiple-unreachables.ll
@@ -10,64 +10,62 @@ define amdgpu_kernel void @kernel(i32 %a, ptr addrspace(1) %x, i32 noundef %n) {
; handle multiple-exits CFG. This should be correctly structurized.
; CHECK-LABEL: kernel:
; CHECK: ; %bb.0: ; %entry
-; CHECK-NEXT: s_load_dword s0, s[8:9], 0x10
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
; CHECK-NEXT: s_load_dword s10, s[8:9], 0x0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_cmpk_lg_i32 s0, 0x100
+; CHECK-NEXT: s_cmpk_lg_i32 s2, 0x100
; CHECK-NEXT: s_cbranch_scc0 .LBB0_6
; CHECK-NEXT: ; %bb.1: ; %if.else
; CHECK-NEXT: v_cmp_gt_u32_e32 vcc, 10, v0
+; CHECK-NEXT: s_mov_b64 s[6:7], 0
; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b64 s[2:3], 0
-; CHECK-NEXT: s_mov_b64 s[0:1], 0
-; CHECK-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; CHECK-NEXT: s_and_saveexec_b64 s[8:9], vcc
; CHECK-NEXT: s_cbranch_execz .LBB0_5
; CHECK-NEXT: ; %bb.2: ; %if.then3
; CHECK-NEXT: s_cmp_lg_u32 s10, 0
; CHECK-NEXT: s_cbranch_scc1 .LBB0_14
; CHECK-NEXT: ; %bb.3:
-; CHECK-NEXT: s_mov_b64 s[0:1], -1
-; CHECK-NEXT: .LBB0_4: ; %Flow3
-; CHECK-NEXT: s_and_b64 s[0:1], s[0:1], exec
+; CHECK-NEXT: s_mov_b64 s[2:3], -1
+; CHECK-NEXT: .LBB0_4: ; %Flow5
; CHECK-NEXT: s_and_b64 s[2:3], s[2:3], exec
-; CHECK-NEXT: .LBB0_5: ; %Flow2
-; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
-; CHECK-NEXT: s_and_b64 vcc, exec, s[4:5]
+; CHECK-NEXT: s_and_b64 s[4:5], s[4:5], exec
+; CHECK-NEXT: .LBB0_5: ; %Flow4
+; CHECK-NEXT: s_or_b64 exec, exec, s[8:9]
+; CHECK-NEXT: s_and_b64 vcc, exec, s[6:7]
; CHECK-NEXT: s_cbranch_vccz .LBB0_8
; CHECK-NEXT: s_branch .LBB0_7
; CHECK-NEXT: .LBB0_6:
+; CHECK-NEXT: s_mov_b64 s[4:5], 0
; CHECK-NEXT: s_mov_b64 s[2:3], 0
-; CHECK-NEXT: s_mov_b64 s[0:1], 0
; CHECK-NEXT: s_cbranch_execz .LBB0_8
; CHECK-NEXT: .LBB0_7: ; %if.then
; CHECK-NEXT: s_cmp_lg_u32 s10, 0
-; CHECK-NEXT: s_mov_b64 s[0:1], -1
+; CHECK-NEXT: s_mov_b64 s[2:3], -1
; CHECK-NEXT: s_cbranch_scc1 .LBB0_13
-; CHECK-NEXT: .LBB0_8: ; %Flow4
-; CHECK-NEXT: s_and_saveexec_b64 s[4:5], s[2:3]
+; CHECK-NEXT: .LBB0_8: ; %Flow6
+; CHECK-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; CHECK-NEXT: .LBB0_9: ; %UnifiedUnreachableBlock
; CHECK-NEXT: ; divergent unreachable
-; CHECK-NEXT: .LBB0_10: ; %Flow6
-; CHECK-NEXT: s_or_b64 exec, exec, s[4:5]
-; CHECK-NEXT: s_and_saveexec_b64 s[2:3], s[0:1]
+; CHECK-NEXT: .LBB0_10: ; %Flow8
+; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
+; CHECK-NEXT: s_and_saveexec_b64 s[4:5], s[2:3]
; CHECK-NEXT: s_cbranch_execz .LBB0_12
; CHECK-NEXT: ; %bb.11: ; %if.end6.sink.split
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x8
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CHECK-NEXT: v_mov_b32_e32 v1, s10
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: global_store_dword v0, v1, s[0:1]
; CHECK-NEXT: .LBB0_12: ; %UnifiedReturnBlock
; CHECK-NEXT: s_endpgm
; CHECK-NEXT: .LBB0_13: ; %cond.false
-; CHECK-NEXT: s_mov_b64 s[0:1], 0
-; CHECK-NEXT: s_or_b64 s[2:3], s[2:3], exec
+; CHECK-NEXT: s_mov_b64 s[2:3], 0
+; CHECK-NEXT: s_or_b64 s[4:5], s[4:5], exec
; CHECK-NEXT: s_trap 2
-; CHECK-NEXT: s_and_saveexec_b64 s[4:5], s[2:3]
+; CHECK-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; CHECK-NEXT: s_cbranch_execnz .LBB0_9
; CHECK-NEXT: s_branch .LBB0_10
; CHECK-NEXT: .LBB0_14: ; %cond.false.i8
-; CHECK-NEXT: s_mov_b64 s[2:3], -1
+; CHECK-NEXT: s_mov_b64 s[4:5], -1
; CHECK-NEXT: s_trap 2
; CHECK-NEXT: s_branch .LBB0_4
; UNIFY-LABEL: @kernel(
@@ -134,26 +132,26 @@ if.end6:
define amdgpu_kernel void @kernel_callbr(i32 %a, ptr addrspace(1) %x, i32 noundef %n) {
; CHECK-LABEL: kernel_callbr:
; CHECK: ; %bb.0: ; %entry
-; CHECK-NEXT: s_load_dword s1, s[8:9], 0x10
-; CHECK-NEXT: s_load_dword s0, s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x8
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_cmpk_eq_i32 s1, 0x100
-; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; CHECK-NEXT: s_load_dword s3, s[8:9], 0x0
+; CHECK-NEXT: ; kill: killed $sgpr8_sgpr9
+; CHECK-NEXT: s_cmpk_eq_i32 s2, 0x100
+; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: ; %bb.1: ; %if.then
-; CHECK-NEXT: s_cmp_eq_u32 s0, 0
-; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_cmp_eq_u32 s3, 0
+; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: .LBB1_2: ; %if.end6.sink.split
-; CHECK-NEXT: s_load_dwordx2 s[2:3], s[8:9], 0x8
; CHECK-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; CHECK-NEXT: v_mov_b32_e32 v1, s0
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_dword v0, v1, s[2:3]
+; CHECK-NEXT: v_mov_b32_e32 v1, s3
+; CHECK-NEXT: global_store_dword v0, v1, s[0:1]
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: .LBB1_3: ; Inline asm indirect target
@@ -168,9 +166,10 @@ define amdgpu_kernel void @kernel_callbr(i32 %a, ptr addrspace(1) %x, i32 nounde
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: ; %bb.5: ; %if.then3
-; CHECK-NEXT: s_cmp_eq_u32 s0, 0
-; CHECK-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[2:3]
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_cmp_eq_u32 s3, 0
+; CHECK-NEXT: s_cselect_b64 s[4:5], -1, 0
+; CHECK-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[4:5]
; CHECK-NEXT: ;;#ASMSTART
; CHECK-NEXT: ;;#ASMEND
; CHECK-NEXT: s_branch .LBB1_2
@@ -189,12 +188,12 @@ define amdgpu_kernel void @kernel_callbr(i32 %a, ptr addrspace(1) %x, i32 nounde
; UNIFY-NEXT: [[CMP:%.*]] = icmp eq i32 [[N:%.*]], 256
; UNIFY-NEXT: [[CMP32:%.*]] = zext i1 [[CMP]] to i32
; UNIFY-NEXT: callbr void asm "", "r,!i"(i32 [[CMP32]])
-; UNIFY-NEXT: to label [[IF_THEN:%.*]] [label %if.else]
+; UNIFY-NEXT: to label [[IF_THEN:%.*]] [label [[IF_ELSE:%.*]]]
; UNIFY: if.then:
; UNIFY-NEXT: [[CMP1:%.*]] = icmp eq i32 [[A:%.*]], 0
; UNIFY-NEXT: [[CMP1_32:%.*]] = zext i1 [[CMP1]] to i32
; UNIFY-NEXT: callbr void asm "", "r,!i"(i32 [[CMP1_32]])
-; UNIFY-NEXT: to label [[IF_END6_SINK_SPLIT:%.*]] [label %cond.false]
+; UNIFY-NEXT: to label [[IF_END6_SINK_SPLIT:%.*]] [label [[COND_FALSE:%.*]]]
; UNIFY: cond.false:
; UNIFY-NEXT: call void @llvm.trap()
; UNIFY-NEXT: unreachable
@@ -202,12 +201,12 @@ define amdgpu_kernel void @kernel_callbr(i32 %a, ptr addrspace(1) %x, i32 nounde
; UNIFY-NEXT: [[CMP2:%.*]] = icmp ult i32 [[TID]], 10
; UNIFY-NEXT: [[CMP2_32:%.*]] = zext i1 [[CMP2]] to i32
; UNIFY-NEXT: callbr void asm "", "r,!i"(i32 [[CMP2_32]])
-; UNIFY-NEXT: to label [[IF_THEN3:%.*]] [label %if.end6]
+; UNIFY-NEXT: to label [[IF_THEN3:%.*]] [label [[IF_END6:%.*]]]
; UNIFY: if.then3:
; UNIFY-NEXT: [[CMP1_I7:%.*]] = icmp eq i32 [[A]], 0
; UNIFY-NEXT: [[CMP1_I7_32:%.*]] = zext i1 [[CMP1_I7]] to i32
; UNIFY-NEXT: callbr void asm "", "r,!i"(i32 [[CMP1_I7_32]])
-; UNIFY-NEXT: to label [[IF_END6_SINK_SPLIT]] [label %cond.false.i8]
+; UNIFY-NEXT: to label [[IF_END6_SINK_SPLIT]] [label [[COND_FALSE_I8:%.*]]]
; UNIFY: cond.false.i8:
; UNIFY-NEXT: call void @llvm.trap()
; UNIFY-NEXT: unreachable
@@ -215,7 +214,7 @@ define amdgpu_kernel void @kernel_callbr(i32 %a, ptr addrspace(1) %x, i32 nounde
; UNIFY-NEXT: [[X1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[X:%.*]], i32 [[TID]]
; UNIFY-NEXT: store i32 [[A]], ptr addrspace(1) [[X1]], align 4
; UNIFY-NEXT: callbr void asm "", ""()
-; UNIFY-NEXT: to label [[IF_END6:%.*]] []
+; UNIFY-NEXT: to label [[IF_END6]] []
; UNIFY: if.end6:
; UNIFY-NEXT: ret void
;
diff --git a/llvm/test/CodeGen/AMDGPU/sign_extend.ll b/llvm/test/CodeGen/AMDGPU/sign_extend.ll
index ece46b59ba49e..cec08a27593ac 100644
--- a/llvm/test/CodeGen/AMDGPU/sign_extend.ll
+++ b/llvm/test/CodeGen/AMDGPU/sign_extend.ll
@@ -6,27 +6,29 @@ define amdgpu_kernel void @s_sext_i1_to_i32(ptr addrspace(1) %out, i32 %a, i32 %
; SI-LABEL: s_sext_i1_to_i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_cmp_eq_u32 s4, s5
-; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_cmp_eq_u32 s2, s3
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_sext_i1_to_i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: s_cmp_eq_u32 s4, s5
-; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_cmp_eq_u32 s2, s3
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
%cmp = icmp eq i32 %a, %b
%sext = sext i1 %cmp to i32
@@ -37,13 +39,15 @@ define amdgpu_kernel void @s_sext_i1_to_i32(ptr addrspace(1) %out, i32 %a, i32 %
define amdgpu_kernel void @test_s_sext_i32_to_i64(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) nounwind {
; SI-LABEL: test_s_sext_i32_to_i64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dword s8, s[4:5], 0xd
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mul_i32 s0, s0, s1
-; SI-NEXT: s_add_i32 s0, s0, s2
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mul_i32 s0, s2, s3
+; SI-NEXT: s_add_i32 s0, s0, s8
+; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: s_ashr_i32 s1, s0, 31
; SI-NEXT: v_mov_b32_e32 v0, s0
; SI-NEXT: v_mov_b32_e32 v1, s1
@@ -52,13 +56,15 @@ define amdgpu_kernel void @test_s_sext_i32_to_i64(ptr addrspace(1) %out, i32 %a,
;
; VI-LABEL: test_s_sext_i32_to_i64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dword s8, s[4:5], 0x34
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mul_i32 s0, s0, s1
-; VI-NEXT: s_add_i32 s0, s0, s2
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mul_i32 s0, s2, s3
+; VI-NEXT: s_add_i32 s0, s0, s8
+; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: s_ashr_i32 s1, s0, 31
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -76,29 +82,31 @@ define amdgpu_kernel void @s_sext_i1_to_i64(ptr addrspace(1) %out, i32 %a, i32 %
; SI-LABEL: s_sext_i1_to_i64:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_cmp_eq_u32 s4, s5
-; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_cmp_eq_u32 s2, s3
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
; SI-NEXT: v_mov_b32_e32 v1, v0
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_sext_i1_to_i64:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: s_cmp_eq_u32 s4, s5
-; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_cmp_eq_u32 s2, s3
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
; VI-NEXT: v_mov_b32_e32 v1, v0
-; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
%cmp = icmp eq i32 %a, %b
%sext = sext i1 %cmp to i64
@@ -109,28 +117,30 @@ define amdgpu_kernel void @s_sext_i1_to_i64(ptr addrspace(1) %out, i32 %a, i32 %
define amdgpu_kernel void @s_sext_i32_to_i64(ptr addrspace(1) %out, i32 %a) nounwind {
; SI-LABEL: s_sext_i32_to_i64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_ashr_i32 s4, s6, 31
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: v_mov_b32_e32 v1, s4
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_ashr_i32 s0, s2, 31
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: v_mov_b32_e32 v1, s0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_sext_i32_to_i64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ashr_i32 s4, s6, 31
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: v_mov_b32_e32 v1, s4
-; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_ashr_i32 s0, s2, 31
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v1, s0
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
%sext = sext i32 %a to i64
store i64 %sext, ptr addrspace(1) %out, align 8
@@ -182,28 +192,30 @@ define amdgpu_kernel void @v_sext_i32_to_i64(ptr addrspace(1) %out, ptr addrspac
define amdgpu_kernel void @s_sext_i16_to_i64(ptr addrspace(1) %out, i16 %a) nounwind {
; SI-LABEL: s_sext_i16_to_i64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_bfe_i64 s[4:5], s[6:7], 0x100000
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_bfe_i64 s[0:1], s[2:3], 0x100000
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_sext_i16_to_i64:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bfe_i64 s[4:5], s[6:7], 0x100000
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_bfe_i64 s[0:1], s[2:3], 0x100000
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
%sext = sext i16 %a to i64
store i64 %sext, ptr addrspace(1) %out, align 8
@@ -214,27 +226,29 @@ define amdgpu_kernel void @s_sext_i1_to_i16(ptr addrspace(1) %out, i32 %a, i32 %
; SI-LABEL: s_sext_i1_to_i16:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_cmp_eq_u32 s4, s5
-; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_cmp_eq_u32 s2, s3
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_sext_i1_to_i16:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: s_cmp_eq_u32 s4, s5
-; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
-; VI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; VI-NEXT: s_cmp_eq_u32 s2, s3
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
+; VI-NEXT: buffer_store_short v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
%cmp = icmp eq i32 %a, %b
%sext = sext i1 %cmp to i16
@@ -249,14 +263,16 @@ define amdgpu_kernel void @s_sext_i1_to_i16(ptr addrspace(1) %out, i32 %a, i32 %
define amdgpu_kernel void @s_sext_i1_to_i16_with_and(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c, i32 %d) nounwind {
; SI-LABEL: s_sext_i1_to_i16_with_and:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_eq_u32 s0, s1
-; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
; SI-NEXT: s_cmp_eq_u32 s2, s3
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: s_cmp_eq_u32 s8, s9
; SI-NEXT: s_cselect_b64 s[2:3], -1, 0
; SI-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
@@ -265,14 +281,16 @@ define amdgpu_kernel void @s_sext_i1_to_i16_with_and(ptr addrspace(1) %out, i32
;
; VI-LABEL: s_sext_i1_to_i16_with_and:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0x34
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s0, s1
-; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
; VI-NEXT: s_cmp_eq_u32 s2, s3
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: s_cmp_eq_u32 s8, s9
; VI-NEXT: s_cselect_b64 s[2:3], -1, 0
; VI-NEXT: s_and_b64 s[0:1], s[0:1], s[2:3]
; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
@@ -289,13 +307,15 @@ define amdgpu_kernel void @s_sext_i1_to_i16_with_and(ptr addrspace(1) %out, i32
define amdgpu_kernel void @v_sext_i1_to_i16_with_and(ptr addrspace(1) %out, i32 %a, i32 %b, i32 %c) nounwind {
; SI-LABEL: v_sext_i1_to_i16_with_and:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dword s8, s[4:5], 0xd
; SI-NEXT: s_mov_b32 s7, 0xf000
; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_eq_u32 s1, s2
-; SI-NEXT: v_cmp_eq_u32_e32 vcc, s0, v0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_cmp_eq_u32 s3, s8
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
; SI-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
@@ -304,13 +324,15 @@ define amdgpu_kernel void @v_sext_i1_to_i16_with_and(ptr addrspace(1) %out, i32
;
; VI-LABEL: v_sext_i1_to_i16_with_and:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_load_dword s8, s[4:5], 0x34
; VI-NEXT: s_mov_b32 s7, 0xf000
; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s1, s2
-; VI-NEXT: v_cmp_eq_u32_e32 vcc, s0, v0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_cmp_eq_u32 s3, s8
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: v_cmp_eq_u32_e32 vcc, s2, v0
; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
; VI-NEXT: s_and_b64 s[0:1], vcc, s[0:1]
; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
@@ -336,51 +358,53 @@ define amdgpu_kernel void @v_sext_i1_to_i16_with_and(ptr addrspace(1) %out, i32
define amdgpu_kernel void @s_sext_v4i8_to_v4i32(ptr addrspace(1) %out, i32 %a) nounwind {
; SI-LABEL: s_sext_v4i8_to_v4i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_ashr_i32 s4, s6, 24
-; SI-NEXT: s_bfe_i32 s5, s6, 0x80010
-; SI-NEXT: s_bfe_i32 s7, s6, 0x80008
-; SI-NEXT: s_sext_i32_i8 s6, s6
-; SI-NEXT: v_mov_b32_e32 v0, s6
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_ashr_i32 s0, s2, 24
+; SI-NEXT: s_bfe_i32 s1, s2, 0x80010
+; SI-NEXT: s_bfe_i32 s3, s2, 0x80008
+; SI-NEXT: s_sext_i32_i8 s2, s2
+; SI-NEXT: v_mov_b32_e32 v0, s2
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s7
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: v_mov_b32_e32 v0, s3
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s5
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: v_mov_b32_e32 v0, s1
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_sext_v4i8_to_v4i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ashr_i32 s4, s6, 24
-; VI-NEXT: s_bfe_i32 s5, s6, 0x80010
-; VI-NEXT: s_bfe_i32 s7, s6, 0x80008
-; VI-NEXT: s_sext_i32_i8 s6, s6
-; VI-NEXT: v_mov_b32_e32 v0, s6
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_ashr_i32 s0, s2, 24
+; VI-NEXT: s_bfe_i32 s1, s2, 0x80010
+; VI-NEXT: s_bfe_i32 s3, s2, 0x80008
+; VI-NEXT: s_sext_i32_i8 s2, s2
+; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s7
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: v_mov_b32_e32 v0, s3
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s5
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: v_mov_b32_e32 v0, s1
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_waitcnt vmcnt(0)
; VI-NEXT: s_endpgm
%cast = bitcast i32 %a to <4 x i8>
diff --git a/llvm/test/CodeGen/AMDGPU/siloadstoreopt-misaligned-regsequence.ll b/llvm/test/CodeGen/AMDGPU/siloadstoreopt-misaligned-regsequence.ll
index 39f594d375b50..436823a7d7f08 100644
--- a/llvm/test/CodeGen/AMDGPU/siloadstoreopt-misaligned-regsequence.ll
+++ b/llvm/test/CodeGen/AMDGPU/siloadstoreopt-misaligned-regsequence.ll
@@ -5,12 +5,12 @@ define amdgpu_kernel void @foo(ptr %0) {
; CHECK-LABEL: foo:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; CHECK-NEXT: v_mov_b32_e32 v4, 0
-; CHECK-NEXT: v_mov_b32_e32 v5, 0
-; CHECK-NEXT: v_mov_b32_e32 v6, 0
+; CHECK-NEXT: v_mov_b32_e32 v2, 0
+; CHECK-NEXT: v_mov_b32_e32 v3, v2
+; CHECK-NEXT: v_mov_b32_e32 v4, v2
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_mov_b64_e32 v[2:3], s[0:1]
-; CHECK-NEXT: flat_store_dwordx3 v[2:3], v[4:6]
+; CHECK-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
+; CHECK-NEXT: flat_store_dwordx3 v[0:1], v[2:4]
; CHECK-NEXT: s_endpgm
entry:
%1 = getelementptr inbounds i8, ptr %0, i64 4
diff --git a/llvm/test/CodeGen/AMDGPU/simulated-trap-pseudo-expand.ll b/llvm/test/CodeGen/AMDGPU/simulated-trap-pseudo-expand.ll
index 9796db4717e7b..9d7c9db84a0ef 100644
--- a/llvm/test/CodeGen/AMDGPU/simulated-trap-pseudo-expand.ll
+++ b/llvm/test/CodeGen/AMDGPU/simulated-trap-pseudo-expand.ll
@@ -6,24 +6,21 @@ declare void @llvm.trap()
define amdgpu_kernel void @simulated_trap_pseudo_expand(i64 %offset, i1 %should_continue, ptr %base) {
; CHECK-LABEL: simulated_trap_pseudo_expand:
; CHECK: ; %bb.0: ; %entry
-; CHECK-NEXT: s_load_b32 s0, s[4:5], 0x8
+; CHECK-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: s_bitcmp1_b32 s0, 0
-; CHECK-NEXT: s_cselect_b32 s0, -1, 0
+; CHECK-NEXT: s_bitcmp1_b32 s2, 0
+; CHECK-NEXT: s_cselect_b32 s2, -1, 0
; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; CHECK-NEXT: s_and_b32 vcc_lo, exec_lo, s0
+; CHECK-NEXT: s_and_b32 vcc_lo, exec_lo, s2
; CHECK-NEXT: s_cbranch_vccz .LBB0_2
; CHECK-NEXT: ; %bb.1: ; %normal_path
-; CHECK-NEXT: s_clause 0x1
-; CHECK-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; CHECK-NEXT: s_load_b64 s[2:3], s[4:5], 0x10
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_lshl_b64 s[0:1], s[0:1], 3
-; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_add_u32 s0, s2, s0
; CHECK-NEXT: s_addc_u32 s1, s3, s1
+; CHECK-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; CHECK-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v3, s1
-; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
; CHECK-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v1, v0
; CHECK-NEXT: flat_store_b64 v[2:3], v[0:1]
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll b/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
index 101787abf8ea7..20708d29f8f96 100644
--- a/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/sint_to_fp.f64.ll
@@ -6,38 +6,22 @@
declare i32 @llvm.amdgcn.workitem.id.x() nounwind readnone
define amdgpu_kernel void @sint_to_fp_i32_to_f64(ptr addrspace(1) %out, i32 %in) {
-; CI-LABEL: sint_to_fp_i32_to_f64:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_cvt_f64_i32_e32 v[0:1], s2
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: sint_to_fp_i32_to_f64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cvt_f64_i32_e32 v[0:1], s2
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: sint_to_fp_i32_to_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_cvt_f64_i32_e32 v[0:1], s2
+; GCN-NEXT: v_mov_b32_e32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: sint_to_fp_i32_to_f64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_cvt_f64_i32_e32 v[0:1], s2
@@ -51,44 +35,25 @@ define amdgpu_kernel void @sint_to_fp_i32_to_f64(ptr addrspace(1) %out, i32 %in)
; We can't fold the SGPRs into v_cndmask_b32_e64, because it already
; uses an SGPR (implicit vcc).
define amdgpu_kernel void @sint_to_fp_i1_f64(ptr addrspace(1) %out, i32 %in) {
-; CI-LABEL: sint_to_fp_i1_f64:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_cmp_eq_u32 s2, 0
-; CI-NEXT: s_cselect_b32 s2, 0xbff00000, 0
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v0, 0
-; CI-NEXT: v_mov_b32_e32 v1, s2
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: sint_to_fp_i1_f64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s2, 0
-; VI-NEXT: s_cselect_b32 s2, 0xbff00000, 0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: sint_to_fp_i1_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, 0xbff00000, 0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: sint_to_fp_i1_f64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_eq_u32 s2, 0
@@ -103,44 +68,25 @@ define amdgpu_kernel void @sint_to_fp_i1_f64(ptr addrspace(1) %out, i32 %in) {
}
define amdgpu_kernel void @sint_to_fp_i1_f64_load(ptr addrspace(1) %out, i1 %in) {
-; CI-LABEL: sint_to_fp_i1_f64_load:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_bitcmp1_b32 s2, 0
-; CI-NEXT: s_cselect_b64 s[2:3], -1, 0
-; CI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[2:3]
-; CI-NEXT: v_cvt_f64_i32_e32 v[0:1], v0
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: sint_to_fp_i1_f64_load:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitcmp1_b32 s2, 0
-; VI-NEXT: s_cselect_b64 s[2:3], -1, 0
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[2:3]
-; VI-NEXT: v_cvt_f64_i32_e32 v[0:1], v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: sint_to_fp_i1_f64_load:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_bitcmp1_b32 s2, 0
+; GCN-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[2:3]
+; GCN-NEXT: v_cvt_f64_i32_e32 v[0:1], v0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: sint_to_fp_i1_f64_load:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_bitcmp1_b32 s2, 0
@@ -273,40 +219,23 @@ define amdgpu_kernel void @v_sint_to_fp_i64_to_f64(ptr addrspace(1) %out, ptr ad
; FIXME: bfe and sext on VI+
define amdgpu_kernel void @s_sint_to_fp_i8_to_f64(ptr addrspace(1) %out, i8 %in) {
-; CI-LABEL: s_sint_to_fp_i8_to_f64:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_sext_i32_i8 s2, s2
-; CI-NEXT: v_cvt_f64_i32_e32 v[0:1], s2
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: s_sint_to_fp_i8_to_f64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_sext_i32_i8 s2, s2
-; VI-NEXT: v_cvt_f64_i32_e32 v[0:1], s2
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: s_sint_to_fp_i8_to_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_sext_i32_i8 s2, s2
+; GCN-NEXT: v_cvt_f64_i32_e32 v[0:1], s2
+; GCN-NEXT: v_mov_b32_e32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: s_sint_to_fp_i8_to_f64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_sext_i32_i8 s2, s2
@@ -346,44 +275,25 @@ define double @v_sint_to_fp_i8_to_f64(i8 %in) {
}
define amdgpu_kernel void @s_select_sint_to_fp_i1_vals_f64(ptr addrspace(1) %out, i32 %in) {
-; CI-LABEL: s_select_sint_to_fp_i1_vals_f64:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_cmp_eq_u32 s2, 0
-; CI-NEXT: s_cselect_b32 s2, 0xbff00000, 0
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v0, 0
-; CI-NEXT: v_mov_b32_e32 v1, s2
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: s_select_sint_to_fp_i1_vals_f64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s2, 0
-; VI-NEXT: s_cselect_b32 s2, 0xbff00000, 0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: s_select_sint_to_fp_i1_vals_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, 0xbff00000, 0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: s_select_sint_to_fp_i1_vals_f64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_eq_u32 s2, 0
@@ -427,44 +337,25 @@ define void @v_select_sint_to_fp_i1_vals_f64(ptr addrspace(1) %out, i32 %in) {
}
define amdgpu_kernel void @s_select_sint_to_fp_i1_vals_i64(ptr addrspace(1) %out, i32 %in) {
-; CI-LABEL: s_select_sint_to_fp_i1_vals_i64:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_cmp_eq_u32 s2, 0
-; CI-NEXT: s_cselect_b32 s2, 0xbff00000, 0
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v0, 0
-; CI-NEXT: v_mov_b32_e32 v1, s2
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: s_select_sint_to_fp_i1_vals_i64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s2, 0
-; VI-NEXT: s_cselect_b32 s2, 0xbff00000, 0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: s_select_sint_to_fp_i1_vals_i64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, 0xbff00000, 0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: s_select_sint_to_fp_i1_vals_i64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_eq_u32 s2, 0
@@ -539,44 +430,25 @@ define void @v_swap_select_sint_to_fp_i1_vals_f64(ptr addrspace(1) %out, i32 %in
; TODO: This should swap the selected order / invert the compare and do it.
define amdgpu_kernel void @s_swap_select_sint_to_fp_i1_vals_f64(ptr addrspace(1) %out, i32 %in) {
-; CI-LABEL: s_swap_select_sint_to_fp_i1_vals_f64:
-; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[8:9], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; CI-NEXT: s_add_i32 s12, s12, s17
-; CI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; CI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_cmp_eq_u32 s2, 0
-; CI-NEXT: s_cselect_b32 s2, 0, 0xbff00000
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v0, 0
-; CI-NEXT: v_mov_b32_e32 v1, s2
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; CI-NEXT: s_endpgm
-;
-; VI-LABEL: s_swap_select_sint_to_fp_i1_vals_f64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s2, 0
-; VI-NEXT: s_cselect_b32 s2, 0, 0xbff00000
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: s_swap_select_sint_to_fp_i1_vals_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, 0, 0xbff00000
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: s_swap_select_sint_to_fp_i1_vals_f64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_eq_u32 s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
index 76f8f484fc763..1143fbc037242 100644
--- a/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sminmax.v2i16.ll
@@ -7,8 +7,7 @@
define amdgpu_kernel void @s_abs_v2i16(ptr addrspace(1) %out, <2 x i16> %val) #0 {
; GFX9-LABEL: s_abs_v2i16:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_pk_sub_i16 v1, 0, s2
@@ -19,52 +18,52 @@ define amdgpu_kernel void @s_abs_v2i16(ptr addrspace(1) %out, <2 x i16> %val) #0
;
; VI-LABEL: s_abs_v2i16:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_sub_i32 s3, 0, s2
-; VI-NEXT: s_lshr_b32 s4, s2, 16
-; VI-NEXT: s_ashr_i32 s5, s2, 16
-; VI-NEXT: s_sub_i32 s4, 0, s4
-; VI-NEXT: s_sext_i32_i16 s3, s3
-; VI-NEXT: s_sext_i32_i16 s2, s2
-; VI-NEXT: s_sext_i32_i16 s4, s4
-; VI-NEXT: s_max_i32 s2, s2, s3
-; VI-NEXT: s_max_i32 s4, s5, s4
-; VI-NEXT: s_add_i32 s2, s2, 2
-; VI-NEXT: s_lshl_b32 s3, s4, 16
-; VI-NEXT: s_and_b32 s2, s2, 0xffff
-; VI-NEXT: s_or_b32 s2, s3, s2
-; VI-NEXT: s_add_i32 s2, s2, 0x20000
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_sub_i32 s0, 0, s2
+; VI-NEXT: s_lshr_b32 s1, s2, 16
+; VI-NEXT: s_ashr_i32 s3, s2, 16
+; VI-NEXT: s_sub_i32 s1, 0, s1
+; VI-NEXT: s_sext_i32_i16 s0, s0
+; VI-NEXT: s_sext_i32_i16 s2, s2
+; VI-NEXT: s_sext_i32_i16 s1, s1
+; VI-NEXT: s_max_i32 s0, s2, s0
+; VI-NEXT: s_max_i32 s1, s3, s1
+; VI-NEXT: s_add_i32 s0, s0, 2
+; VI-NEXT: s_lshl_b32 s1, s1, 16
+; VI-NEXT: s_and_b32 s0, s0, 0xffff
+; VI-NEXT: s_or_b32 s0, s1, s0
+; VI-NEXT: s_add_i32 s0, s0, 0x20000
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; CI-LABEL: s_abs_v2i16:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s6, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s5, s6, 16
-; CI-NEXT: s_ashr_i32 s4, s6, 16
-; CI-NEXT: s_sext_i32_i16 s7, s6
-; CI-NEXT: s_sub_i32 s6, 0, s6
-; CI-NEXT: s_sub_i32 s5, 0, s5
-; CI-NEXT: s_sext_i32_i16 s6, s6
-; CI-NEXT: s_sext_i32_i16 s5, s5
-; CI-NEXT: s_max_i32 s4, s4, s5
-; CI-NEXT: s_max_i32 s5, s7, s6
-; CI-NEXT: s_add_i32 s5, s5, 2
-; CI-NEXT: s_lshl_b32 s4, s4, 16
-; CI-NEXT: s_and_b32 s5, s5, 0xffff
-; CI-NEXT: s_or_b32 s4, s4, s5
-; CI-NEXT: s_add_i32 s4, s4, 0x20000
-; CI-NEXT: v_mov_b32_e32 v0, s4
-; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; CI-NEXT: s_mov_b32 s5, s1
+; CI-NEXT: s_lshr_b32 s1, s2, 16
+; CI-NEXT: s_mov_b32 s4, s0
+; CI-NEXT: s_ashr_i32 s0, s2, 16
+; CI-NEXT: s_sext_i32_i16 s3, s2
+; CI-NEXT: s_sub_i32 s2, 0, s2
+; CI-NEXT: s_sub_i32 s1, 0, s1
+; CI-NEXT: s_sext_i32_i16 s2, s2
+; CI-NEXT: s_sext_i32_i16 s1, s1
+; CI-NEXT: s_max_i32 s0, s0, s1
+; CI-NEXT: s_max_i32 s1, s3, s2
+; CI-NEXT: s_add_i32 s1, s1, 2
+; CI-NEXT: s_lshl_b32 s0, s0, 16
+; CI-NEXT: s_and_b32 s1, s1, 0xffff
+; CI-NEXT: s_or_b32 s0, s0, s1
+; CI-NEXT: s_add_i32 s0, s0, 0x20000
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
%neg = sub <2 x i16> zeroinitializer, %val
%cond = icmp sgt <2 x i16> %val, %neg
@@ -156,8 +155,7 @@ define amdgpu_kernel void @v_abs_v2i16(ptr addrspace(1) %out, ptr addrspace(1) %
define amdgpu_kernel void @s_abs_v2i16_2(ptr addrspace(1) %out, <2 x i16> %val) #0 {
; GFX9-LABEL: s_abs_v2i16_2:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_pk_sub_i16 v1, 0, s2
@@ -168,52 +166,52 @@ define amdgpu_kernel void @s_abs_v2i16_2(ptr addrspace(1) %out, <2 x i16> %val)
;
; VI-LABEL: s_abs_v2i16_2:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_sub_i32 s3, 0, s2
-; VI-NEXT: s_lshr_b32 s4, s2, 16
-; VI-NEXT: s_ashr_i32 s5, s2, 16
-; VI-NEXT: s_sub_i32 s4, 0, s4
-; VI-NEXT: s_sext_i32_i16 s3, s3
-; VI-NEXT: s_sext_i32_i16 s2, s2
-; VI-NEXT: s_sext_i32_i16 s4, s4
-; VI-NEXT: s_max_i32 s2, s2, s3
-; VI-NEXT: s_max_i32 s4, s5, s4
-; VI-NEXT: s_add_i32 s2, s2, 2
-; VI-NEXT: s_lshl_b32 s3, s4, 16
-; VI-NEXT: s_and_b32 s2, s2, 0xffff
-; VI-NEXT: s_or_b32 s2, s3, s2
-; VI-NEXT: s_add_i32 s2, s2, 0x20000
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: s_sub_i32 s0, 0, s2
+; VI-NEXT: s_lshr_b32 s1, s2, 16
+; VI-NEXT: s_ashr_i32 s3, s2, 16
+; VI-NEXT: s_sub_i32 s1, 0, s1
+; VI-NEXT: s_sext_i32_i16 s0, s0
+; VI-NEXT: s_sext_i32_i16 s2, s2
+; VI-NEXT: s_sext_i32_i16 s1, s1
+; VI-NEXT: s_max_i32 s0, s2, s0
+; VI-NEXT: s_max_i32 s1, s3, s1
+; VI-NEXT: s_add_i32 s0, s0, 2
+; VI-NEXT: s_lshl_b32 s1, s1, 16
+; VI-NEXT: s_and_b32 s0, s0, 0xffff
+; VI-NEXT: s_or_b32 s0, s1, s0
+; VI-NEXT: s_add_i32 s0, s0, 0x20000
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
; CI-LABEL: s_abs_v2i16_2:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s6, s[4:5], 0xb
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, -1
+; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_lshr_b32 s5, s6, 16
-; CI-NEXT: s_ashr_i32 s4, s6, 16
-; CI-NEXT: s_sext_i32_i16 s7, s6
-; CI-NEXT: s_sub_i32 s6, 0, s6
-; CI-NEXT: s_sub_i32 s5, 0, s5
-; CI-NEXT: s_sext_i32_i16 s6, s6
-; CI-NEXT: s_sext_i32_i16 s5, s5
-; CI-NEXT: s_max_i32 s4, s4, s5
-; CI-NEXT: s_max_i32 s5, s7, s6
-; CI-NEXT: s_add_i32 s5, s5, 2
-; CI-NEXT: s_lshl_b32 s4, s4, 16
-; CI-NEXT: s_and_b32 s5, s5, 0xffff
-; CI-NEXT: s_or_b32 s4, s4, s5
-; CI-NEXT: s_add_i32 s4, s4, 0x20000
-; CI-NEXT: v_mov_b32_e32 v0, s4
-; CI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; CI-NEXT: s_mov_b32 s5, s1
+; CI-NEXT: s_lshr_b32 s1, s2, 16
+; CI-NEXT: s_mov_b32 s4, s0
+; CI-NEXT: s_ashr_i32 s0, s2, 16
+; CI-NEXT: s_sext_i32_i16 s3, s2
+; CI-NEXT: s_sub_i32 s2, 0, s2
+; CI-NEXT: s_sub_i32 s1, 0, s1
+; CI-NEXT: s_sext_i32_i16 s2, s2
+; CI-NEXT: s_sext_i32_i16 s1, s1
+; CI-NEXT: s_max_i32 s0, s0, s1
+; CI-NEXT: s_max_i32 s1, s3, s2
+; CI-NEXT: s_add_i32 s1, s1, 2
+; CI-NEXT: s_lshl_b32 s0, s0, 16
+; CI-NEXT: s_and_b32 s1, s1, 0xffff
+; CI-NEXT: s_or_b32 s0, s0, s1
+; CI-NEXT: s_add_i32 s0, s0, 0x20000
+; CI-NEXT: v_mov_b32_e32 v0, s0
+; CI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; CI-NEXT: s_endpgm
%z0 = insertelement <2 x i16> poison, i16 0, i16 0
%z1 = insertelement <2 x i16> %z0, i16 0, i16 1
@@ -331,79 +329,80 @@ define amdgpu_kernel void @s_abs_v4i16(ptr addrspace(1) %out, <4 x i16> %val) #0
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_lshr_b32 s7, s2, 16
-; VI-NEXT: s_sub_i32 s7, 0, s7
-; VI-NEXT: s_sub_i32 s4, 0, s3
-; VI-NEXT: s_lshr_b32 s6, s3, 16
-; VI-NEXT: s_ashr_i32 s8, s2, 16
-; VI-NEXT: s_sext_i32_i16 s7, s7
-; VI-NEXT: s_sub_i32 s5, 0, s2
-; VI-NEXT: s_sub_i32 s6, 0, s6
-; VI-NEXT: s_max_i32 s7, s8, s7
-; VI-NEXT: s_ashr_i32 s8, s3, 16
-; VI-NEXT: s_sext_i32_i16 s4, s4
-; VI-NEXT: s_sext_i32_i16 s3, s3
-; VI-NEXT: s_sext_i32_i16 s6, s6
-; VI-NEXT: s_sext_i32_i16 s5, s5
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_sub_i32 s1, 0, s2
+; VI-NEXT: s_lshr_b32 s5, s2, 16
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_sub_i32 s0, 0, s3
+; VI-NEXT: s_lshr_b32 s4, s3, 16
+; VI-NEXT: s_sub_i32 s5, 0, s5
+; VI-NEXT: s_ashr_i32 s6, s2, 16
+; VI-NEXT: s_sext_i32_i16 s1, s1
; VI-NEXT: s_sext_i32_i16 s2, s2
-; VI-NEXT: s_max_i32 s3, s3, s4
-; VI-NEXT: s_max_i32 s6, s8, s6
-; VI-NEXT: s_max_i32 s2, s2, s5
-; VI-NEXT: s_add_i32 s3, s3, 2
-; VI-NEXT: s_lshl_b32 s4, s6, 16
-; VI-NEXT: s_and_b32 s3, s3, 0xffff
-; VI-NEXT: s_add_i32 s2, s2, 2
-; VI-NEXT: s_or_b32 s3, s4, s3
-; VI-NEXT: s_lshl_b32 s4, s7, 16
-; VI-NEXT: s_and_b32 s2, s2, 0xffff
-; VI-NEXT: s_or_b32 s2, s4, s2
-; VI-NEXT: s_add_i32 s3, s3, 0x20000
-; VI-NEXT: s_add_i32 s2, s2, 0x20000
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, s2
-; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: s_sub_i32 s4, 0, s4
+; VI-NEXT: s_sext_i32_i16 s5, s5
+; VI-NEXT: s_max_i32 s1, s2, s1
+; VI-NEXT: s_sext_i32_i16 s0, s0
+; VI-NEXT: s_sext_i32_i16 s2, s3
+; VI-NEXT: s_max_i32 s5, s6, s5
+; VI-NEXT: s_ashr_i32 s6, s3, 16
+; VI-NEXT: s_sext_i32_i16 s4, s4
+; VI-NEXT: s_max_i32 s0, s2, s0
+; VI-NEXT: s_max_i32 s4, s6, s4
+; VI-NEXT: s_add_i32 s0, s0, 2
+; VI-NEXT: s_lshl_b32 s2, s4, 16
+; VI-NEXT: s_and_b32 s0, s0, 0xffff
+; VI-NEXT: s_add_i32 s1, s1, 2
+; VI-NEXT: s_or_b32 s0, s2, s0
+; VI-NEXT: s_lshl_b32 s2, s5, 16
+; VI-NEXT: s_and_b32 s1, s1, 0xffff
+; VI-NEXT: s_or_b32 s1, s2, s1
+; VI-NEXT: s_add_i32 s0, s0, 0x20000
+; VI-NEXT: s_add_i32 s1, s1, 0x20000
+; VI-NEXT: v_mov_b32_e32 v2, s1
+; VI-NEXT: v_mov_b32_e32 v3, s0
+; VI-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; VI-NEXT: s_endpgm
;
; CI-LABEL: s_abs_v4i16:
; CI: ; %bb.0:
; CI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; CI-NEXT: s_mov_b32 s7, 0xf000
+; CI-NEXT: s_mov_b32 s6, -1
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; CI-NEXT: s_ashr_i32 s6, s5, 16
-; CI-NEXT: s_lshr_b32 s9, s5, 16
-; CI-NEXT: s_sext_i32_i16 s10, s5
-; CI-NEXT: s_sub_i32 s5, 0, s5
-; CI-NEXT: s_ashr_i32 s7, s4, 16
-; CI-NEXT: s_lshr_b32 s8, s4, 16
-; CI-NEXT: s_sext_i32_i16 s11, s4
-; CI-NEXT: s_sext_i32_i16 s5, s5
-; CI-NEXT: s_sub_i32 s4, 0, s4
+; CI-NEXT: s_mov_b32 s4, s0
+; CI-NEXT: s_mov_b32 s5, s1
+; CI-NEXT: s_ashr_i32 s0, s3, 16
+; CI-NEXT: s_ashr_i32 s1, s2, 16
+; CI-NEXT: s_lshr_b32 s8, s2, 16
+; CI-NEXT: s_lshr_b32 s9, s3, 16
+; CI-NEXT: s_sext_i32_i16 s10, s3
+; CI-NEXT: s_sext_i32_i16 s11, s2
+; CI-NEXT: s_sub_i32 s3, 0, s3
+; CI-NEXT: s_sub_i32 s2, 0, s2
+; CI-NEXT: s_sext_i32_i16 s3, s3
+; CI-NEXT: s_sext_i32_i16 s2, s2
; CI-NEXT: s_sub_i32 s9, 0, s9
-; CI-NEXT: s_sext_i32_i16 s4, s4
-; CI-NEXT: s_sext_i32_i16 s9, s9
; CI-NEXT: s_sub_i32 s8, 0, s8
-; CI-NEXT: s_max_i32 s5, s10, s5
+; CI-NEXT: s_sext_i32_i16 s9, s9
; CI-NEXT: s_sext_i32_i16 s8, s8
-; CI-NEXT: s_max_i32 s6, s6, s9
-; CI-NEXT: s_max_i32 s4, s11, s4
-; CI-NEXT: s_add_i32 s5, s5, 2
-; CI-NEXT: s_max_i32 s7, s7, s8
-; CI-NEXT: s_lshl_b32 s6, s6, 16
-; CI-NEXT: s_and_b32 s5, s5, 0xffff
-; CI-NEXT: s_add_i32 s4, s4, 2
-; CI-NEXT: s_or_b32 s5, s6, s5
-; CI-NEXT: s_lshl_b32 s6, s7, 16
-; CI-NEXT: s_and_b32 s4, s4, 0xffff
-; CI-NEXT: s_or_b32 s4, s6, s4
-; CI-NEXT: s_add_i32 s5, s5, 0x20000
-; CI-NEXT: s_add_i32 s4, s4, 0x20000
-; CI-NEXT: s_mov_b32 s3, 0xf000
-; CI-NEXT: s_mov_b32 s2, -1
-; CI-NEXT: v_mov_b32_e32 v0, s4
-; CI-NEXT: v_mov_b32_e32 v1, s5
-; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; CI-NEXT: s_max_i32 s2, s11, s2
+; CI-NEXT: s_max_i32 s3, s10, s3
+; CI-NEXT: s_max_i32 s1, s1, s8
+; CI-NEXT: s_max_i32 s0, s0, s9
+; CI-NEXT: s_add_i32 s3, s3, 2
+; CI-NEXT: s_add_i32 s2, s2, 2
+; CI-NEXT: s_lshl_b32 s0, s0, 16
+; CI-NEXT: s_and_b32 s3, s3, 0xffff
+; CI-NEXT: s_lshl_b32 s1, s1, 16
+; CI-NEXT: s_and_b32 s2, s2, 0xffff
+; CI-NEXT: s_or_b32 s0, s0, s3
+; CI-NEXT: s_or_b32 s1, s1, s2
+; CI-NEXT: s_add_i32 s0, s0, 0x20000
+; CI-NEXT: s_add_i32 s1, s1, 0x20000
+; CI-NEXT: v_mov_b32_e32 v0, s1
+; CI-NEXT: v_mov_b32_e32 v1, s0
+; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; CI-NEXT: s_endpgm
%z0 = insertelement <4 x i16> poison, i16 0, i16 0
%z1 = insertelement <4 x i16> %z0, i16 0, i16 1
diff --git a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
index c0c1763d54cc0..8dd16aed15aad 100644
--- a/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
+++ b/llvm/test/CodeGen/AMDGPU/splitkit-getsubrangeformask.ll
@@ -44,7 +44,7 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_SUB_I32_1:%[0-9]+]]:sreg_32 = S_SUB_I32 [[S_BUFFER_LOAD_DWORD_IMM]], 30, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], [[S_LSHL_B32_2]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_]], 16, 0 :: (invariant load (s128) from %ir.83, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_]], 16, 0 :: (invariant load (s128) from %ir.88, addrspace 4)
; CHECK-NEXT: early-clobber %73:sgpr_256 = S_LOAD_DWORDX8_IMM_ec undef %74:sreg_64, 0, 0 :: (invariant load (s256) from `ptr addrspace(4) poison`, align 16, addrspace 4)
; CHECK-NEXT: KILL [[S_ADD_U32_]].sub0, [[S_ADD_U32_]].sub1
; CHECK-NEXT: KILL undef %74:sreg_64
@@ -62,14 +62,14 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: undef [[S_ADD_U32_2:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], [[S_LSHL_B32_1]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_2:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_1]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: undef [[S_ADD_U32_3:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], [[S_LSHL_B32_2]], implicit-def $scc
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM1:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_1]], 64, 0 :: (invariant load (s128) from %ir.89, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM2:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_2]], 64, 0 :: (invariant load (s128) from %ir.95, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM1:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_1]], 64, 0 :: (invariant load (s128) from %ir.94, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM2:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_2]], 64, 0 :: (invariant load (s128) from %ir.100, addrspace 4)
; CHECK-NEXT: KILL [[S_ADD_U32_2]].sub0, [[S_ADD_U32_2]].sub1
; CHECK-NEXT: KILL [[S_ADD_U32_1]].sub0, [[S_ADD_U32_1]].sub1
; CHECK-NEXT: [[S_ADD_U32_3:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: [[S_ASHR_I32_3:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 undef %174:sreg_32, 31, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_4:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY6]], undef %174:sreg_32, implicit-def $scc
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM3:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_3]], 64, 0 :: (invariant load (s128) from %ir.101, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM3:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_3]], 64, 0 :: (invariant load (s128) from %ir.106, addrspace 4)
; CHECK-NEXT: [[S_ADD_U32_4:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %54:sreg_32, [[S_ASHR_I32_3]], implicit-def dead $scc, implicit $scc
; CHECK-NEXT: undef [[S_ADD_U32_5:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY7]].sub0, [[S_LSHL_B32_]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_5:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %51:sreg_32, [[S_ASHR_I32_]], implicit-def dead $scc, implicit $scc
@@ -93,10 +93,10 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM2:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[S_MOV_B32_]], 16, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %362:sgpr_128, undef %363:sreg_32, 0, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM3:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM undef %373:sgpr_128, 16, 0 :: (dereferenceable invariant load (s32))
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM4:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_4]], 64, 0 :: (invariant load (s128) from %ir.109, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM5:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_5]], 0, 0 :: (invariant load (s128) from %ir.114, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM6:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_6]], 0, 0 :: (invariant load (s128) from %ir.119, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM7:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_7]], 0, 0 :: (invariant load (s128) from %ir.126, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM4:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_4]], 64, 0 :: (invariant load (s128) from %ir.114, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM5:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_5]], 0, 0 :: (invariant load (s128) from %ir.119, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM6:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_6]], 0, 0 :: (invariant load (s128) from %ir.124, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM7:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_7]], 0, 0 :: (invariant load (s128) from %ir.131, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN2:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM1]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %357:sgpr_128, [[S_ADD_I32_]], 0, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM5:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %368:sgpr_128, [[S_ADD_I32_1]], 0, 0 :: (dereferenceable invariant load (s32))
@@ -117,13 +117,13 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_LSHL4_ADD_U32_:%[0-9]+]]:sreg_32 = S_LSHL4_ADD_U32 [[COPY12]], 16, implicit-def dead $scc
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_SGPR_IMM6:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_SGPR_IMM undef %388:sgpr_128, [[S_LSHL4_ADD_U32_]], 0, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN5:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM4]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM8:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_5]], 224, 0 :: (invariant load (s128) from %ir.131, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM9:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY7]], 224, 0 :: (invariant load (s128) from %ir.147, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM10:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_5]], 576, 0 :: (invariant load (s128) from %ir.152, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM8:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_5]], 224, 0 :: (invariant load (s128) from %ir.136, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM9:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[COPY7]], 224, 0 :: (invariant load (s128) from %ir.152, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM10:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_5]], 576, 0 :: (invariant load (s128) from %ir.157, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN6:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM5]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM11:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_6]], 224, 0 :: (invariant load (s128) from %ir.136, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM12:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_7]], 576, 0 :: (invariant load (s128) from %ir.164, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM13:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_8]], 224, 0 :: (invariant load (s128) from %ir.142, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM11:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_6]], 224, 0 :: (invariant load (s128) from %ir.141, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM12:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_7]], 576, 0 :: (invariant load (s128) from %ir.169, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM13:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_8]], 224, 0 :: (invariant load (s128) from %ir.147, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN7:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM6]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN8:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM7]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[S_ADD_I32_6:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_SGPR_IMM4]], -217, implicit-def dead $scc
@@ -146,22 +146,22 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_ASHR_I32_5:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[S_LSHL_B32_4]], 31, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_18:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY]].sub0, [[S_LSHL_B32_4]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_18:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %57:sreg_32, [[S_ASHR_I32_5]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[S_ADD_U32_18]], 168, 0 :: (invariant load (s32) from %ir.276, align 8, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM14:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_8]], 576, 0 :: (invariant load (s128) from %ir.159, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[S_ADD_U32_18]], 168, 0 :: (invariant load (s32) from %ir.281, align 8, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM14:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_8]], 576, 0 :: (invariant load (s128) from %ir.164, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN11:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM13]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN12:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM9]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN13:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM10]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]].sub3:sgpr_128 = S_MOV_B32 553734060
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]].sub2:sgpr_128 = S_MOV_B32 -1
; CHECK-NEXT: [[COPY15:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX2_IMM]]
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM15:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_9]], 0, 0 :: (invariant load (s128) from %ir.172, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM15:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_9]], 0, 0 :: (invariant load (s128) from %ir.177, addrspace 4)
; CHECK-NEXT: [[COPY15:%[0-9]+]].sub1:sgpr_128 = COPY [[S_MOV_B32_]].sub1
; CHECK-NEXT: [[COPY15:%[0-9]+]].sub0:sgpr_128 = COPY [[S_LOAD_DWORD_IMM]]
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM4:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[COPY15]], 0, 0 :: (dereferenceable invariant load (s32))
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN14:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM14]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN15:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM12]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM16:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_10]], 0, 0 :: (invariant load (s128) from %ir.180, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM17:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_11]], 0, 0 :: (invariant load (s128) from %ir.185, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM16:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_10]], 0, 0 :: (invariant load (s128) from %ir.185, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM17:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_11]], 0, 0 :: (invariant load (s128) from %ir.190, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN16:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM15]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[S_LSHL_B32_5:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY4]], 3, implicit-def dead $scc
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFSET1:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET %73.sub0_sub1_sub2_sub3, 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
@@ -169,14 +169,14 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_ADD_I32_14:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_IMM4]], -467, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_19:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY]].sub0, [[S_LSHL_B32_5]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_19:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %57:sreg_32, [[S_ASHR_I32_6]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[S_ADD_U32_19]], 168, 0 :: (invariant load (s64) from %ir.285, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM1:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[S_ADD_U32_19]], 168, 0 :: (invariant load (s64) from %ir.290, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFSET2:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[S_LOAD_DWORDX4_IMM16]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_DWORD_OFFSET3:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_DWORD_OFFSET [[S_LOAD_DWORDX4_IMM17]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM18:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_12]], 0, 0 :: (invariant load (s128) from %ir.207, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM19:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_13]], 0, 0 :: (invariant load (s128) from %ir.213, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM18:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_12]], 0, 0 :: (invariant load (s128) from %ir.212, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM19:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_13]], 0, 0 :: (invariant load (s128) from %ir.218, addrspace 4)
; CHECK-NEXT: [[COPY16:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX2_IMM]]
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM20:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_14]], 0, 0 :: (invariant load (s128) from %ir.218, addrspace 4)
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM21:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_15]], 0, 0 :: (invariant load (s128) from %ir.223, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM20:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_14]], 0, 0 :: (invariant load (s128) from %ir.223, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM21:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_15]], 0, 0 :: (invariant load (s128) from %ir.228, addrspace 4)
; CHECK-NEXT: [[S_AND_B32_:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_LOAD_DWORDX2_IMM1]].sub1, 65535, implicit-def dead $scc
; CHECK-NEXT: [[COPY16:%[0-9]+]].sub0:sgpr_128 = COPY [[S_LOAD_DWORDX2_IMM1]].sub0
; CHECK-NEXT: [[COPY16:%[0-9]+]].sub1:sgpr_128 = COPY [[S_AND_B32_]]
@@ -190,20 +190,20 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_ADD_I32_15:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_IMM5]], -468, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_20:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY]].sub0, [[S_LSHL_B32_6]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_20:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %57:sreg_32, [[S_ASHR_I32_7]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM2:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[S_ADD_U32_20]], 168, 0 :: (invariant load (s64) from %ir.296, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM2:%[0-9]+]]:sreg_64_xexec = S_LOAD_DWORDX2_IMM [[S_ADD_U32_20]], 168, 0 :: (invariant load (s64) from %ir.303, addrspace 4)
; CHECK-NEXT: [[COPY17:%[0-9]+]]:sgpr_128 = COPY [[S_LOAD_DWORDX2_IMM]]
; CHECK-NEXT: [[S_AND_B32_1:%[0-9]+]]:sreg_32 = S_AND_B32 [[S_LOAD_DWORDX2_IMM2]].sub1, 65535, implicit-def dead $scc
; CHECK-NEXT: [[COPY17:%[0-9]+]].sub0:sgpr_128 = COPY [[S_LOAD_DWORDX2_IMM2]].sub0
; CHECK-NEXT: [[COPY17:%[0-9]+]].sub1:sgpr_128 = COPY [[S_AND_B32_1]]
; CHECK-NEXT: [[S_BUFFER_LOAD_DWORD_IMM6:%[0-9]+]]:sreg_32_xm0_xexec = S_BUFFER_LOAD_DWORD_IMM [[COPY17]], 0, 0 :: (dereferenceable invariant load (s32))
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM22:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_16]], 160, 0 :: (invariant load (s128) from %ir.259, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM22:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_16]], 160, 0 :: (invariant load (s128) from %ir.264, addrspace 4)
; CHECK-NEXT: [[S_LSHL_B32_7:%[0-9]+]]:sreg_32 = S_LSHL_B32 [[COPY14]], 3, implicit-def dead $scc
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM23:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_17]], 160, 0 :: (invariant load (s128) from %ir.268, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM23:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_17]], 160, 0 :: (invariant load (s128) from %ir.273, addrspace 4)
; CHECK-NEXT: [[S_ASHR_I32_8:%[0-9]+]]:sreg_32_xm0 = S_ASHR_I32 [[S_LSHL_B32_7]], 31, implicit-def dead $scc
; CHECK-NEXT: [[S_ADD_I32_16:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_IMM6]], -469, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_21:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY]].sub0, [[S_LSHL_B32_7]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_21:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %57:sreg_32, [[S_ASHR_I32_8]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: [[S_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[S_ADD_U32_21]], 168, 0 :: (invariant load (s32) from %ir.308, align 8, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORD_IMM1:%[0-9]+]]:sreg_32_xm0_xexec = S_LOAD_DWORD_IMM [[S_ADD_U32_21]], 168, 0 :: (invariant load (s32) from %ir.317, align 8, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN21:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM22]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN22:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM23]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: KILL [[S_LOAD_DWORDX4_IMM23]]
@@ -221,13 +221,13 @@ define amdgpu_gs void @_amdgpu_gs_main(i32 inreg %primShaderTableAddrLow, <31 x
; CHECK-NEXT: [[S_ADD_I32_22:%[0-9]+]]:sreg_32 = S_ADD_I32 [[S_BUFFER_LOAD_DWORD_IMM7]], -473, implicit-def dead $scc
; CHECK-NEXT: undef [[S_ADD_U32_22:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY1]], [[S_LSHL_B32_]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_22:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %33:sreg_32, [[S_ASHR_I32_]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM24:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_22]], 96, 0 :: (invariant load (s128) from %ir.326, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM24:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_22]], 96, 0 :: (invariant load (s128) from %ir.335, addrspace 4)
; CHECK-NEXT: undef [[S_ADD_U32_23:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY1]], [[S_LSHL_B32_1]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_23:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %33:sreg_32, [[S_ASHR_I32_1]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM25:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_23]], 96, 0 :: (invariant load (s128) from %ir.332, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM25:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_23]], 96, 0 :: (invariant load (s128) from %ir.341, addrspace 4)
; CHECK-NEXT: undef [[S_ADD_U32_24:%[0-9]+]].sub0:sreg_64 = S_ADD_U32 [[COPY1]], [[S_LSHL_B32_2]], implicit-def $scc
; CHECK-NEXT: [[S_ADD_U32_24:%[0-9]+]].sub1:sreg_64 = S_ADDC_U32 undef %33:sreg_32, [[S_ASHR_I32_2]], implicit-def dead $scc, implicit $scc
- ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM26:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_24]], 96, 0 :: (invariant load (s128) from %ir.338, addrspace 4)
+ ; CHECK-NEXT: [[S_LOAD_DWORDX4_IMM26:%[0-9]+]]:sgpr_128 = S_LOAD_DWORDX4_IMM [[S_ADD_U32_24]], 96, 0 :: (invariant load (s128) from %ir.347, addrspace 4)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN23:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM24]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN24:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM25]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
; CHECK-NEXT: [[BUFFER_LOAD_FORMAT_X_IDXEN25:%[0-9]+]]:vgpr_32 = BUFFER_LOAD_FORMAT_X_IDXEN [[V_MOV_B32_e32_]], [[S_LOAD_DWORDX4_IMM26]], 0, 0, 0, 0, implicit $exec :: (dereferenceable load (s32), align 1, addrspace 8)
diff --git a/llvm/test/CodeGen/AMDGPU/sra.ll b/llvm/test/CodeGen/AMDGPU/sra.ll
index c7e7927c37da3..72ec39e07c4a7 100644
--- a/llvm/test/CodeGen/AMDGPU/sra.ll
+++ b/llvm/test/CodeGen/AMDGPU/sra.ll
@@ -392,30 +392,32 @@ define amdgpu_kernel void @ashr_v4i16(ptr addrspace(1) %out, ptr addrspace(1) %i
define amdgpu_kernel void @s_ashr_i64(ptr addrspace(1) %out, i32 %in) {
; SI-LABEL: s_ashr_i64:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_ashr_i32 s7, s6, 31
-; SI-NEXT: s_ashr_i64 s[4:5], s[6:7], 8
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: v_mov_b32_e32 v1, s5
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; SI-NEXT: s_ashr_i32 s3, s2, 31
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_ashr_i64 s[0:1], s[2:3], 8
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: v_mov_b32_e32 v1, s1
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_ashr_i64:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_ashr_i32 s7, s6, 31
-; VI-NEXT: s_ashr_i64 s[4:5], s[6:7], 8
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: v_mov_b32_e32 v1, s5
-; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; VI-NEXT: s_ashr_i32 s3, s2, 31
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_ashr_i64 s[0:1], s[2:3], 8
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; EG-LABEL: s_ashr_i64:
diff --git a/llvm/test/CodeGen/AMDGPU/store-to-constant.ll b/llvm/test/CodeGen/AMDGPU/store-to-constant.ll
index 9b3b52012f327..60cb98e6afa7b 100644
--- a/llvm/test/CodeGen/AMDGPU/store-to-constant.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-to-constant.ll
@@ -7,8 +7,7 @@
define amdgpu_kernel void @store_as4_i8(ptr addrspace(4) %p, i8 %v) {
; CHECK-LABEL: store_as4_i8:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_load_dword s2, s[8:9], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s2
@@ -21,8 +20,7 @@ define amdgpu_kernel void @store_as4_i8(ptr addrspace(4) %p, i8 %v) {
define amdgpu_kernel void @store_as4_i16(ptr addrspace(4) %p, i16 %v) {
; CHECK-LABEL: store_as4_i16:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_load_dword s2, s[8:9], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s2
@@ -35,8 +33,7 @@ define amdgpu_kernel void @store_as4_i16(ptr addrspace(4) %p, i16 %v) {
define amdgpu_kernel void @store_as4_i32(ptr addrspace(4) %p, i32 %v) {
; CHECK-LABEL: store_as4_i32:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_load_dword s2, s[8:9], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s2
@@ -63,8 +60,7 @@ define amdgpu_kernel void @store_as4_i64(ptr addrspace(4) %p, i64 %v) {
define amdgpu_kernel void @store_as4_float(ptr addrspace(4) %p, float %v) {
; CHECK-LABEL: store_as4_float:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_load_dword s2, s[8:9], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s2
@@ -91,8 +87,7 @@ define amdgpu_kernel void @store_as4_double(ptr addrspace(4) %p, double %v) {
define amdgpu_kernel void @store_as4_half(ptr addrspace(4) %p, half %v) {
; CHECK-LABEL: store_as4_half:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_load_dword s2, s[8:9], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s2
@@ -105,8 +100,7 @@ define amdgpu_kernel void @store_as4_half(ptr addrspace(4) %p, half %v) {
define amdgpu_kernel void @store_as4_2xi8(ptr addrspace(4) %p, <2 x i8> %v) {
; CHECK-LABEL: store_as4_2xi8:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_load_dword s2, s[8:9], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s2
@@ -119,8 +113,7 @@ define amdgpu_kernel void @store_as4_2xi8(ptr addrspace(4) %p, <2 x i8> %v) {
define amdgpu_kernel void @store_as4_2xi16(ptr addrspace(4) %p, <2 x i16> %v) {
; CHECK-LABEL: store_as4_2xi16:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_load_dword s2, s[8:9], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s2
@@ -136,7 +129,8 @@ define amdgpu_kernel void @store_as4_2xi32(ptr addrspace(4) %p, <2 x i32> %v) {
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; CHECK-NEXT: v_mov_b32_e32 v2, s2
+; CHECK-NEXT: v_mov_b32_e32 v3, s3
; CHECK-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; CHECK-NEXT: s_endpgm
store <2 x i32> %v, ptr addrspace(4) %p
@@ -146,8 +140,7 @@ define amdgpu_kernel void @store_as4_2xi32(ptr addrspace(4) %p, <2 x i32> %v) {
define amdgpu_kernel void @store_as4_2xhalf(ptr addrspace(4) %p, <2 x half> %v) {
; CHECK-LABEL: store_as4_2xhalf:
; CHECK: ; %bb.0:
-; CHECK-NEXT: s_load_dword s2, s[8:9], 0x8
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
+; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v1, s2
@@ -163,7 +156,8 @@ define amdgpu_kernel void @store_as4_2xfloat(ptr addrspace(4) %p, <2 x float> %v
; CHECK-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: v_pk_mov_b32 v[2:3], s[2:3], s[2:3] op_sel:[0,1]
+; CHECK-NEXT: v_mov_b32_e32 v2, s2
+; CHECK-NEXT: v_mov_b32_e32 v3, s3
; CHECK-NEXT: global_store_dwordx2 v0, v[2:3], s[0:1]
; CHECK-NEXT: s_endpgm
store <2 x float> %v, ptr addrspace(4) %p
diff --git a/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll b/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
index ade922e8af166..9a8912c8c82b7 100644
--- a/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
+++ b/llvm/test/CodeGen/AMDGPU/structurize-hoist.ll
@@ -85,18 +85,17 @@ merge:
define amdgpu_kernel void @test_loop_with_if( ptr %ptr, i1 %cond) #0 {
; GFX900-LABEL: test_loop_with_if:
; GFX900: ; %bb.0: ; %entry
-; GFX900-NEXT: s_load_dword s0, s[4:5], 0x2c
-; GFX900-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX900-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX900-NEXT: v_mov_b32_e32 v6, 0
-; GFX900-NEXT: s_mov_b64 s[2:3], 0
-; GFX900-NEXT: s_movk_i32 s8, 0xfe
+; GFX900-NEXT: s_mov_b64 s[0:1], 0
; GFX900-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-NEXT: s_bitcmp1_b32 s0, 0
-; GFX900-NEXT: s_cselect_b64 s[0:1], -1, 0
-; GFX900-NEXT: v_mov_b32_e32 v1, s6
-; GFX900-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[0:1]
-; GFX900-NEXT: s_xor_b64 s[0:1], s[0:1], -1
-; GFX900-NEXT: v_mov_b32_e32 v2, s7
+; GFX900-NEXT: s_bitcmp1_b32 s10, 0
+; GFX900-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GFX900-NEXT: v_mov_b32_e32 v1, s8
+; GFX900-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[2:3]
+; GFX900-NEXT: s_xor_b64 s[2:3], s[2:3], -1
+; GFX900-NEXT: v_mov_b32_e32 v2, s9
+; GFX900-NEXT: s_movk_i32 s8, 0xfe
; GFX900-NEXT: s_branch .LBB2_2
; GFX900-NEXT: .LBB2_1: ; %latch
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
@@ -104,21 +103,21 @@ define amdgpu_kernel void @test_loop_with_if( ptr %ptr, i1 %cond) #0 {
; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX900-NEXT: v_add_u32_e32 v6, 20, v3
; GFX900-NEXT: v_cmp_lt_i32_e32 vcc, s8, v6
-; GFX900-NEXT: s_or_b64 s[2:3], vcc, s[2:3]
+; GFX900-NEXT: s_or_b64 s[0:1], vcc, s[0:1]
; GFX900-NEXT: flat_store_dword v[1:2], v3
-; GFX900-NEXT: s_andn2_b64 exec, exec, s[2:3]
+; GFX900-NEXT: s_andn2_b64 exec, exec, s[0:1]
; GFX900-NEXT: s_cbranch_execz .LBB2_8
; GFX900-NEXT: .LBB2_2: ; %loop
; GFX900-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX900-NEXT: flat_load_dwordx2 v[3:4], v[1:2]
; GFX900-NEXT: v_cmp_ne_u32_e32 vcc, 1, v5
-; GFX900-NEXT: s_mov_b64 s[6:7], s[0:1]
+; GFX900-NEXT: s_mov_b64 s[6:7], s[2:3]
; GFX900-NEXT: s_mov_b64 s[4:5], 0
; GFX900-NEXT: s_cbranch_vccnz .LBB2_4
; GFX900-NEXT: ; %bb.3: ; %if
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
; GFX900-NEXT: v_cmp_gt_i32_e32 vcc, 11, v6
-; GFX900-NEXT: s_andn2_b64 s[6:7], s[0:1], exec
+; GFX900-NEXT: s_andn2_b64 s[6:7], s[2:3], exec
; GFX900-NEXT: s_and_b64 s[10:11], vcc, exec
; GFX900-NEXT: s_mov_b64 s[4:5], -1
; GFX900-NEXT: s_or_b64 s[6:7], s[6:7], s[10:11]
@@ -132,7 +131,7 @@ define amdgpu_kernel void @test_loop_with_if( ptr %ptr, i1 %cond) #0 {
; GFX900-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX900-NEXT: v_add_u32_e32 v3, v3, v4
; GFX900-NEXT: s_andn2_b64 s[4:5], s[4:5], exec
-; GFX900-NEXT: .LBB2_6: ; %Flow1
+; GFX900-NEXT: .LBB2_6: ; %Flow2
; GFX900-NEXT: ; in Loop: Header=BB2_2 Depth=1
; GFX900-NEXT: s_or_b64 exec, exec, s[6:7]
; GFX900-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
diff --git a/llvm/test/CodeGen/AMDGPU/sub.ll b/llvm/test/CodeGen/AMDGPU/sub.ll
index 02c70fbd970f8..075d34a8c6281 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.ll
@@ -11,23 +11,24 @@ define amdgpu_kernel void @s_sub_i32(ptr addrspace(1) %out, i32 %a, i32 %b) {
; GFX6-LABEL: s_sub_i32:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GFX6-NEXT: s_sub_i32 s4, s4, s5
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_sub_i32 s0, s2, s3
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: s_sub_i32:
; GFX8: ; %bb.0:
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sub_i32 s2, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_sub_i32 s0, s2, s3
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
@@ -58,32 +59,31 @@ define amdgpu_kernel void @s_sub_i32(ptr addrspace(1) %out, i32 %a, i32 %b) {
define amdgpu_kernel void @s_sub_imm_i32(ptr addrspace(1) %out, i32 %a) {
; GFX6-LABEL: s_sub_imm_i32:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dword s6, s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GFX6-NEXT: s_mov_b32 s3, 0xf000
-; GFX6-NEXT: s_mov_b32 s2, -1
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: s_sub_i32 s4, 0x4d2, s6
-; GFX6-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX6-NEXT: s_mov_b32 s4, s0
+; GFX6-NEXT: s_sub_i32 s0, 0x4d2, s2
+; GFX6-NEXT: s_mov_b32 s5, s1
+; GFX6-NEXT: v_mov_b32_e32 v0, s0
+; GFX6-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: s_sub_imm_i32:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: s_sub_i32 s2, 0x4d2, s2
; GFX8-NEXT: v_mov_b32_e32 v0, s0
+; GFX8-NEXT: s_sub_i32 s0, 0x4d2, s2
; GFX8-NEXT: v_mov_b32_e32 v1, s1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v2, s0
; GFX8-NEXT: flat_store_dword v[0:1], v2
; GFX8-NEXT: s_endpgm
;
; GFX9-LABEL: s_sub_imm_i32:
; GFX9: ; %bb.0:
-; GFX9-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v0, 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: s_sub_i32 s2, 0x4d2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
index 7ce381eeb5a2f..40cfc8e049211 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
@@ -223,39 +223,44 @@ define amdgpu_kernel void @s_test_sub_v2i16_kernarg(ptr addrspace(1) %out, <2 x
; VI-LABEL: s_test_sub_v2i16_kernarg:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: s_lshr_b32 s6, s5, 16
-; VI-NEXT: s_lshr_b32 s7, s4, 16
-; VI-NEXT: s_sub_i32 s4, s4, s5
-; VI-NEXT: s_sub_i32 s5, s7, s6
-; VI-NEXT: s_lshl_b32 s5, s5, 16
-; VI-NEXT: s_and_b32 s4, s4, 0xffff
-; VI-NEXT: s_or_b32 s4, s4, s5
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v0, s4
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_lshr_b32 s0, s3, 16
+; VI-NEXT: s_lshr_b32 s1, s2, 16
+; VI-NEXT: s_sub_i32 s0, s1, s0
+; VI-NEXT: s_sub_i32 s1, s2, s3
+; VI-NEXT: s_lshl_b32 s0, s0, 16
+; VI-NEXT: s_and_b32 s1, s1, 0xffff
+; VI-NEXT: s_or_b32 s0, s1, s0
+; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GFX10-LABEL: s_test_sub_v2i16_kernarg:
; GFX10: ; %bb.0:
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s7, 0x31016000
+; GFX10-NEXT: s_mov_b32 s6, -1
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_pk_sub_i16 v0, s2, s3
-; GFX10-NEXT: s_mov_b32 s3, 0x31016000
-; GFX10-NEXT: s_mov_b32 s2, -1
-; GFX10-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GFX10-NEXT: s_mov_b32 s4, s0
+; GFX10-NEXT: s_mov_b32 s5, s1
+; GFX10-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_test_sub_v2i16_kernarg:
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s7, 0x31016000
+; GFX11-NEXT: s_mov_b32 s6, -1
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_pk_sub_i16 v0, s2, s3
-; GFX11-NEXT: s_mov_b32 s3, 0x31016000
-; GFX11-NEXT: s_mov_b32 s2, -1
-; GFX11-NEXT: buffer_store_b32 v0, off, s[0:3], 0
+; GFX11-NEXT: s_mov_b32 s4, s0
+; GFX11-NEXT: s_mov_b32 s5, s1
+; GFX11-NEXT: buffer_store_b32 v0, off, s[4:7], 0
; GFX11-NEXT: s_endpgm
%add = sub <2 x i16> %a, %b
store <2 x i16> %add, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll b/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
index 4621be5cab450..52706a26c6c11 100644
--- a/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
+++ b/llvm/test/CodeGen/AMDGPU/subreg-coalescer-undef-use.ll
@@ -10,7 +10,7 @@ target triple="amdgcn--"
define amdgpu_kernel void @foobar(float %a0, float %a1, ptr addrspace(1) %out) #1 {
; CHECK-LABEL: foobar:
; CHECK: ; %bb.0: ; %entry
-; CHECK-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x9
+; CHECK-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9
; CHECK-NEXT: v_mbcnt_lo_u32_b32_e64 v0, -1, 0
; CHECK-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; CHECK-NEXT: s_mov_b32 s2, -1
@@ -19,7 +19,7 @@ define amdgpu_kernel void @foobar(float %a0, float %a1, ptr addrspace(1) %out) #
; CHECK-NEXT: v_mov_b32_e32 v1, s5
; CHECK-NEXT: v_mov_b32_e32 v2, s6
; CHECK-NEXT: v_mov_b32_e32 v3, s7
-; CHECK-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; CHECK-NEXT: s_and_saveexec_b64 s[0:1], vcc
; CHECK-NEXT: ; %bb.1: ; %ift
; CHECK-NEXT: s_mov_b32 s4, s5
; CHECK-NEXT: v_mov_b32_e32 v0, s4
@@ -27,10 +27,10 @@ define amdgpu_kernel void @foobar(float %a0, float %a1, ptr addrspace(1) %out) #
; CHECK-NEXT: v_mov_b32_e32 v2, s6
; CHECK-NEXT: v_mov_b32_e32 v3, s7
; CHECK-NEXT: ; %bb.2: ; %ife
-; CHECK-NEXT: s_or_b64 exec, exec, s[6:7]
-; CHECK-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xb
+; CHECK-NEXT: s_or_b64 exec, exec, s[0:1]
; CHECK-NEXT: s_mov_b32 s3, 0xf000
-; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: s_mov_b32 s0, s6
+; CHECK-NEXT: s_mov_b32 s1, s7
; CHECK-NEXT: buffer_store_dword v1, off, s[0:3], 0
; CHECK-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/trunc-store-i1.ll b/llvm/test/CodeGen/AMDGPU/trunc-store-i1.ll
index ecc1defe6acac..b564a07f73501 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc-store-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc-store-i1.ll
@@ -3,8 +3,8 @@
; GCN-LABEL: {{^}}global_truncstore_i32_to_i1:
-; GCN: s_load_dword [[LOAD:s[0-9]+]],
-; GCN: s_and_b32 [[SREG:s[0-9]+]], [[LOAD]], 1
+; GCN: s_load_dwordx4
+; GCN: s_and_b32 [[SREG:s[0-9]+]], [[LOAD:s[0-9]+]], 1
; GCN: v_mov_b32_e32 [[VREG:v[0-9]+]], [[SREG]]
; GCN: buffer_store_byte [[VREG]],
define amdgpu_kernel void @global_truncstore_i32_to_i1(ptr addrspace(1) %out, i32 %val) nounwind {
@@ -23,8 +23,8 @@ define amdgpu_kernel void @global_truncstore_i64_to_i1(ptr addrspace(1) %out, i6
; FIXME: VGPR on VI
; GCN-LABEL: {{^}}s_arg_global_truncstore_i16_to_i1:
-; GCN: s_load_dword [[LOAD:s[0-9]+]],
-; GCN: s_and_b32 [[SREG:s[0-9]+]], [[LOAD]], 1
+; GCN: s_load_dwordx4
+; GCN: s_and_b32 [[SREG:s[0-9]+]], [[LOAD:s[0-9]+]], 1
; GCN: v_mov_b32_e32 [[VREG:v[0-9]+]], [[SREG]]
; GCN: buffer_store_byte [[VREG]],
define amdgpu_kernel void @s_arg_global_truncstore_i16_to_i1(ptr addrspace(1) %out, i16 %val) nounwind {
diff --git a/llvm/test/CodeGen/AMDGPU/trunc.ll b/llvm/test/CodeGen/AMDGPU/trunc.ll
index 76f60f1e5dbfc..d27eed94f655b 100644
--- a/llvm/test/CodeGen/AMDGPU/trunc.ll
+++ b/llvm/test/CodeGen/AMDGPU/trunc.ll
@@ -277,25 +277,25 @@ define amdgpu_kernel void @trunc_i8_to_i1(ptr addrspace(1) %out, ptr addrspace(1
define amdgpu_kernel void @sgpr_trunc_i16_to_i1(ptr addrspace(1) %out, i16 %a) {
; SI-LABEL: sgpr_trunc_i16_to_i1:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_short v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 1
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: sgpr_trunc_i16_to_i1:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 1
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 1
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -333,25 +333,25 @@ define amdgpu_kernel void @sgpr_trunc_i16_to_i1(ptr addrspace(1) %out, i16 %a) {
define amdgpu_kernel void @sgpr_trunc_i32_to_i1(ptr addrspace(1) %out, i32 %a) {
; SI-LABEL: sgpr_trunc_i32_to_i1:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s4, s6, 1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_and_b32 s0, s2, 1
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: sgpr_trunc_i32_to_i1:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 1
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_and_b32 s0, s2, 1
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
index 24b4b79101ea0..86b83f9a27e0b 100644
--- a/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
+++ b/llvm/test/CodeGen/AMDGPU/tuple-allocation-failure.ll
@@ -32,46 +32,45 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-LABEL: kernel:
; GLOBALNESS1: ; %bb.0: ; %bb
; GLOBALNESS1-NEXT: s_mov_b64 s[36:37], s[6:7]
+; GLOBALNESS1-NEXT: s_mov_b64 s[38:39], s[4:5]
; GLOBALNESS1-NEXT: s_load_dwordx4 s[68:71], s[8:9], 0x0
-; GLOBALNESS1-NEXT: s_load_dword s6, s[8:9], 0x14
+; GLOBALNESS1-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x14
; GLOBALNESS1-NEXT: v_mov_b32_e32 v42, 0
; GLOBALNESS1-NEXT: v_pk_mov_b32 v[44:45], 0, 0
; GLOBALNESS1-NEXT: global_store_dword v[44:45], v42, off
; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0)
; GLOBALNESS1-NEXT: global_load_dword v2, v42, s[68:69]
-; GLOBALNESS1-NEXT: s_mov_b64 s[38:39], s[4:5]
-; GLOBALNESS1-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x18
-; GLOBALNESS1-NEXT: s_load_dword s7, s[8:9], 0x20
; GLOBALNESS1-NEXT: s_add_u32 flat_scratch_lo, s12, s17
; GLOBALNESS1-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; GLOBALNESS1-NEXT: s_add_u32 s0, s0, s17
-; GLOBALNESS1-NEXT: v_mov_b32_e32 v41, v0
-; GLOBALNESS1-NEXT: v_mov_b32_e32 v0, 0
-; GLOBALNESS1-NEXT: v_mov_b32_e32 v1, 0x40994400
; GLOBALNESS1-NEXT: s_addc_u32 s1, s1, 0
-; GLOBALNESS1-NEXT: s_waitcnt lgkmcnt(0)
-; GLOBALNESS1-NEXT: v_cmp_ngt_f64_e64 s[50:51], s[4:5], v[0:1]
-; GLOBALNESS1-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[4:5], 0
-; GLOBALNESS1-NEXT: ; implicit-def: $vgpr60 : SGPR spill to VGPR lane
; GLOBALNESS1-NEXT: s_bitcmp1_b32 s70, 0
-; GLOBALNESS1-NEXT: v_writelane_b32 v60, s4, 0
-; GLOBALNESS1-NEXT: v_writelane_b32 v60, s5, 1
-; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS1-NEXT: s_xor_b64 s[52:53], s[4:5], -1
-; GLOBALNESS1-NEXT: s_bitcmp1_b32 s6, 0
-; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v40, 0, 1, s[4:5]
+; GLOBALNESS1-NEXT: s_mov_b64 s[34:35], s[10:11]
+; GLOBALNESS1-NEXT: s_cselect_b64 s[10:11], -1, 0
+; GLOBALNESS1-NEXT: s_xor_b64 s[52:53], s[10:11], -1
+; GLOBALNESS1-NEXT: s_bitcmp1_b32 s4, 0
+; GLOBALNESS1-NEXT: s_mov_b64 s[48:49], s[8:9]
+; GLOBALNESS1-NEXT: s_mov_b32 s8, s5
; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0
+; GLOBALNESS1-NEXT: v_mov_b32_e32 v41, v0
+; GLOBALNESS1-NEXT: v_mov_b32_e32 v0, 0
+; GLOBALNESS1-NEXT: v_mov_b32_e32 v1, 0x40994400
+; GLOBALNESS1-NEXT: s_mov_b32 s9, s6
; GLOBALNESS1-NEXT: s_xor_b64 s[64:65], s[4:5], -1
+; GLOBALNESS1-NEXT: v_cmp_ngt_f64_e64 s[50:51], s[8:9], v[0:1]
+; GLOBALNESS1-NEXT: v_cmp_ngt_f64_e64 s[8:9], s[8:9], 0
+; GLOBALNESS1-NEXT: ; implicit-def: $vgpr60 : SGPR spill to VGPR lane
; GLOBALNESS1-NEXT: s_bitcmp1_b32 s7, 0
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s8, 0
; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0
+; GLOBALNESS1-NEXT: v_writelane_b32 v60, s9, 1
; GLOBALNESS1-NEXT: s_xor_b64 s[66:67], s[4:5], -1
; GLOBALNESS1-NEXT: s_mov_b32 s84, s16
-; GLOBALNESS1-NEXT: s_mov_b64 s[48:49], s[8:9]
; GLOBALNESS1-NEXT: s_mov_b32 s85, s15
; GLOBALNESS1-NEXT: s_mov_b32 s86, s14
-; GLOBALNESS1-NEXT: s_mov_b64 s[34:35], s[10:11]
; GLOBALNESS1-NEXT: v_mov_b32_e32 v46, 0x80
; GLOBALNESS1-NEXT: v_mov_b32_e32 v47, 0
+; GLOBALNESS1-NEXT: v_cndmask_b32_e64 v40, 0, 1, s[10:11]
; GLOBALNESS1-NEXT: v_mov_b32_e32 v43, v42
; GLOBALNESS1-NEXT: s_mov_b32 s32, 0
; GLOBALNESS1-NEXT: ; implicit-def: $vgpr58_vgpr59
@@ -93,11 +92,11 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: v_readlane_b32 s7, v60, 7
; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GLOBALNESS1-NEXT: s_cbranch_vccz .LBB1_28
-; GLOBALNESS1-NEXT: .LBB1_2: ; %Flow15
+; GLOBALNESS1-NEXT: .LBB1_2: ; %Flow17
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[4:5]
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], 0
-; GLOBALNESS1-NEXT: .LBB1_3: ; %Flow28
+; GLOBALNESS1-NEXT: .LBB1_3: ; %Flow30
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[6:7]
; GLOBALNESS1-NEXT: v_pk_mov_b32 v[58:59], v[0:1], v[0:1] op_sel:[0,1]
@@ -135,12 +134,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_cmp_lt_i32 s71, 1
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], -1
; GLOBALNESS1-NEXT: s_cbranch_scc1 .LBB1_7
-; GLOBALNESS1-NEXT: ; %bb.6: ; %LeafBlock12
+; GLOBALNESS1-NEXT: ; %bb.6: ; %LeafBlock14
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: s_cmp_lg_u32 s71, 1
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], 0
; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS1-NEXT: .LBB1_7: ; %Flow26
+; GLOBALNESS1-NEXT: .LBB1_7: ; %Flow28
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GLOBALNESS1-NEXT: s_cbranch_vccnz .LBB1_9
@@ -149,7 +148,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_cmp_lg_u32 s71, 0
; GLOBALNESS1-NEXT: s_mov_b64 s[8:9], 0
; GLOBALNESS1-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS1-NEXT: .LBB1_9: ; %Flow25
+; GLOBALNESS1-NEXT: .LBB1_9: ; %Flow27
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: s_mov_b64 s[6:7], -1
; GLOBALNESS1-NEXT: s_and_b64 vcc, exec, s[4:5]
@@ -185,7 +184,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: s_waitcnt vmcnt(0)
; GLOBALNESS1-NEXT: v_cmp_nlt_f64_e64 s[80:81], 0, v[0:1]
; GLOBALNESS1-NEXT: s_branch .LBB1_16
-; GLOBALNESS1-NEXT: .LBB1_14: ; %Flow16
+; GLOBALNESS1-NEXT: .LBB1_14: ; %Flow18
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[4:5]
; GLOBALNESS1-NEXT: .LBB1_15: ; %bb63.i
@@ -253,13 +252,13 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_16 Depth=2
; GLOBALNESS1-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS1-NEXT: s_branch .LBB1_14
-; GLOBALNESS1-NEXT: .LBB1_24: ; %Flow23
+; GLOBALNESS1-NEXT: .LBB1_24: ; %Flow25
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: v_readlane_b32 s8, v60, 8
; GLOBALNESS1-NEXT: v_pk_mov_b32 v[0:1], 0, 0
; GLOBALNESS1-NEXT: s_mov_b32 s71, s87
; GLOBALNESS1-NEXT: v_readlane_b32 s9, v60, 9
-; GLOBALNESS1-NEXT: .LBB1_25: ; %Flow24
+; GLOBALNESS1-NEXT: .LBB1_25: ; %Flow26
; GLOBALNESS1-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS1-NEXT: s_or_b64 exec, exec, s[98:99]
; GLOBALNESS1-NEXT: s_and_saveexec_b64 s[4:5], s[68:69]
@@ -321,46 +320,45 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-LABEL: kernel:
; GLOBALNESS0: ; %bb.0: ; %bb
; GLOBALNESS0-NEXT: s_mov_b64 s[36:37], s[6:7]
+; GLOBALNESS0-NEXT: s_mov_b64 s[38:39], s[4:5]
; GLOBALNESS0-NEXT: s_load_dwordx4 s[68:71], s[8:9], 0x0
-; GLOBALNESS0-NEXT: s_load_dword s6, s[8:9], 0x14
+; GLOBALNESS0-NEXT: s_load_dwordx4 s[4:7], s[8:9], 0x14
; GLOBALNESS0-NEXT: v_mov_b32_e32 v42, 0
; GLOBALNESS0-NEXT: v_pk_mov_b32 v[44:45], 0, 0
; GLOBALNESS0-NEXT: global_store_dword v[44:45], v42, off
; GLOBALNESS0-NEXT: s_waitcnt lgkmcnt(0)
; GLOBALNESS0-NEXT: global_load_dword v2, v42, s[68:69]
-; GLOBALNESS0-NEXT: s_mov_b64 s[38:39], s[4:5]
-; GLOBALNESS0-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x18
-; GLOBALNESS0-NEXT: s_load_dword s7, s[8:9], 0x20
; GLOBALNESS0-NEXT: s_add_u32 flat_scratch_lo, s12, s17
; GLOBALNESS0-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; GLOBALNESS0-NEXT: s_add_u32 s0, s0, s17
-; GLOBALNESS0-NEXT: v_mov_b32_e32 v41, v0
-; GLOBALNESS0-NEXT: v_mov_b32_e32 v0, 0
-; GLOBALNESS0-NEXT: v_mov_b32_e32 v1, 0x40994400
; GLOBALNESS0-NEXT: s_addc_u32 s1, s1, 0
-; GLOBALNESS0-NEXT: s_waitcnt lgkmcnt(0)
-; GLOBALNESS0-NEXT: v_cmp_ngt_f64_e64 s[50:51], s[4:5], v[0:1]
-; GLOBALNESS0-NEXT: v_cmp_ngt_f64_e64 s[4:5], s[4:5], 0
-; GLOBALNESS0-NEXT: ; implicit-def: $vgpr60 : SGPR spill to VGPR lane
; GLOBALNESS0-NEXT: s_bitcmp1_b32 s70, 0
-; GLOBALNESS0-NEXT: v_writelane_b32 v60, s4, 0
-; GLOBALNESS0-NEXT: v_writelane_b32 v60, s5, 1
-; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS0-NEXT: s_xor_b64 s[52:53], s[4:5], -1
-; GLOBALNESS0-NEXT: s_bitcmp1_b32 s6, 0
-; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v40, 0, 1, s[4:5]
+; GLOBALNESS0-NEXT: s_mov_b64 s[34:35], s[10:11]
+; GLOBALNESS0-NEXT: s_cselect_b64 s[10:11], -1, 0
+; GLOBALNESS0-NEXT: s_xor_b64 s[52:53], s[10:11], -1
+; GLOBALNESS0-NEXT: s_bitcmp1_b32 s4, 0
+; GLOBALNESS0-NEXT: s_mov_b64 s[48:49], s[8:9]
+; GLOBALNESS0-NEXT: s_mov_b32 s8, s5
; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0
+; GLOBALNESS0-NEXT: v_mov_b32_e32 v41, v0
+; GLOBALNESS0-NEXT: v_mov_b32_e32 v0, 0
+; GLOBALNESS0-NEXT: v_mov_b32_e32 v1, 0x40994400
+; GLOBALNESS0-NEXT: s_mov_b32 s9, s6
; GLOBALNESS0-NEXT: s_xor_b64 s[64:65], s[4:5], -1
+; GLOBALNESS0-NEXT: v_cmp_ngt_f64_e64 s[50:51], s[8:9], v[0:1]
+; GLOBALNESS0-NEXT: v_cmp_ngt_f64_e64 s[8:9], s[8:9], 0
+; GLOBALNESS0-NEXT: ; implicit-def: $vgpr60 : SGPR spill to VGPR lane
; GLOBALNESS0-NEXT: s_bitcmp1_b32 s7, 0
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s8, 0
; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0
+; GLOBALNESS0-NEXT: v_writelane_b32 v60, s9, 1
; GLOBALNESS0-NEXT: s_xor_b64 s[66:67], s[4:5], -1
; GLOBALNESS0-NEXT: s_mov_b32 s82, s16
-; GLOBALNESS0-NEXT: s_mov_b64 s[48:49], s[8:9]
; GLOBALNESS0-NEXT: s_mov_b32 s83, s15
; GLOBALNESS0-NEXT: s_mov_b32 s84, s14
-; GLOBALNESS0-NEXT: s_mov_b64 s[34:35], s[10:11]
; GLOBALNESS0-NEXT: v_mov_b32_e32 v46, 0x80
; GLOBALNESS0-NEXT: v_mov_b32_e32 v47, 0
+; GLOBALNESS0-NEXT: v_cndmask_b32_e64 v40, 0, 1, s[10:11]
; GLOBALNESS0-NEXT: v_mov_b32_e32 v43, v42
; GLOBALNESS0-NEXT: s_mov_b32 s32, 0
; GLOBALNESS0-NEXT: ; implicit-def: $vgpr58_vgpr59
@@ -382,11 +380,11 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: v_readlane_b32 s7, v60, 7
; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GLOBALNESS0-NEXT: s_cbranch_vccz .LBB1_28
-; GLOBALNESS0-NEXT: .LBB1_2: ; %Flow15
+; GLOBALNESS0-NEXT: .LBB1_2: ; %Flow17
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[4:5]
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], 0
-; GLOBALNESS0-NEXT: .LBB1_3: ; %Flow28
+; GLOBALNESS0-NEXT: .LBB1_3: ; %Flow30
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[6:7]
; GLOBALNESS0-NEXT: v_pk_mov_b32 v[58:59], v[0:1], v[0:1] op_sel:[0,1]
@@ -424,12 +422,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_cmp_lt_i32 s71, 1
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], -1
; GLOBALNESS0-NEXT: s_cbranch_scc1 .LBB1_7
-; GLOBALNESS0-NEXT: ; %bb.6: ; %LeafBlock12
+; GLOBALNESS0-NEXT: ; %bb.6: ; %LeafBlock14
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: s_cmp_lg_u32 s71, 1
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], 0
; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS0-NEXT: .LBB1_7: ; %Flow26
+; GLOBALNESS0-NEXT: .LBB1_7: ; %Flow28
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: s_andn2_b64 vcc, exec, s[6:7]
; GLOBALNESS0-NEXT: s_cbranch_vccnz .LBB1_9
@@ -438,7 +436,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_cmp_lg_u32 s71, 0
; GLOBALNESS0-NEXT: s_mov_b64 s[8:9], 0
; GLOBALNESS0-NEXT: s_cselect_b64 s[4:5], -1, 0
-; GLOBALNESS0-NEXT: .LBB1_9: ; %Flow25
+; GLOBALNESS0-NEXT: .LBB1_9: ; %Flow27
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: s_mov_b64 s[6:7], -1
; GLOBALNESS0-NEXT: s_and_b64 vcc, exec, s[4:5]
@@ -473,7 +471,7 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: s_waitcnt vmcnt(0)
; GLOBALNESS0-NEXT: v_cmp_nlt_f64_e64 s[80:81], 0, v[0:1]
; GLOBALNESS0-NEXT: s_branch .LBB1_16
-; GLOBALNESS0-NEXT: .LBB1_14: ; %Flow16
+; GLOBALNESS0-NEXT: .LBB1_14: ; %Flow18
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[4:5]
; GLOBALNESS0-NEXT: .LBB1_15: ; %bb63.i
@@ -541,12 +539,12 @@ define amdgpu_kernel void @kernel(ptr addrspace(1) %arg1.global, i1 %tmp3.i.i, i
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_16 Depth=2
; GLOBALNESS0-NEXT: global_store_dwordx2 v[44:45], v[42:43], off
; GLOBALNESS0-NEXT: s_branch .LBB1_14
-; GLOBALNESS0-NEXT: .LBB1_24: ; %Flow23
+; GLOBALNESS0-NEXT: .LBB1_24: ; %Flow25
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: v_readlane_b32 s8, v60, 8
; GLOBALNESS0-NEXT: v_pk_mov_b32 v[0:1], 0, 0
; GLOBALNESS0-NEXT: v_readlane_b32 s9, v60, 9
-; GLOBALNESS0-NEXT: .LBB1_25: ; %Flow24
+; GLOBALNESS0-NEXT: .LBB1_25: ; %Flow26
; GLOBALNESS0-NEXT: ; in Loop: Header=BB1_4 Depth=1
; GLOBALNESS0-NEXT: s_or_b64 exec, exec, s[98:99]
; GLOBALNESS0-NEXT: s_and_saveexec_b64 s[4:5], s[68:69]
diff --git a/llvm/test/CodeGen/AMDGPU/uaddo.ll b/llvm/test/CodeGen/AMDGPU/uaddo.ll
index b000fae124ede..dd00940886051 100644
--- a/llvm/test/CodeGen/AMDGPU/uaddo.ll
+++ b/llvm/test/CodeGen/AMDGPU/uaddo.ll
@@ -867,112 +867,109 @@ define amdgpu_kernel void @v_uaddo_v2i32(ptr addrspace(1) %out, ptr addrspace(1)
define amdgpu_kernel void @s_uaddo_clamp_bit(ptr addrspace(1) %out, ptr addrspace(1) %carryout, i32 %a, i32 %b) #0 {
; SI-LABEL: s_uaddo_clamp_bit:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s1
-; SI-NEXT: s_cmp_eq_u32 s0, s1
-; SI-NEXT: v_add_i32_e32 v0, vcc, s0, v0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: v_mov_b32_e32 v0, s7
+; SI-NEXT: s_cmp_eq_u32 s6, s7
+; SI-NEXT: v_add_i32_e32 v0, vcc, s6, v0
; SI-NEXT: s_cbranch_scc1 .LBB8_2
; SI-NEXT: ; %bb.1: ; %if
-; SI-NEXT: s_xor_b64 s[0:1], vcc, -1
+; SI-NEXT: s_xor_b64 s[4:5], vcc, -1
; SI-NEXT: .LBB8_2: ; %exit
-; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s11, 0xf000
; SI-NEXT: s_mov_b32 s10, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s8, s4
-; SI-NEXT: s_mov_b32 s9, s5
-; SI-NEXT: s_mov_b32 s4, s6
-; SI-NEXT: s_mov_b32 s5, s7
-; SI-NEXT: s_mov_b32 s6, s10
-; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_mov_b32 s0, s2
+; SI-NEXT: s_mov_b32 s1, s3
+; SI-NEXT: s_mov_b32 s2, s10
+; SI-NEXT: s_mov_b32 s3, s11
; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_uaddo_clamp_bit:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s1
-; VI-NEXT: s_cmp_eq_u32 s0, s1
-; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0
-; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s7
+; VI-NEXT: s_cmp_eq_u32 s6, s7
+; VI-NEXT: v_add_u32_e32 v0, vcc, s6, v0
; VI-NEXT: s_cbranch_scc1 .LBB8_2
; VI-NEXT: ; %bb.1: ; %if
-; VI-NEXT: s_xor_b64 s[0:1], vcc, -1
+; VI-NEXT: s_xor_b64 s[4:5], vcc, -1
; VI-NEXT: .LBB8_2: ; %exit
-; VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
-; VI-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[0:1]
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s4
-; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: v_mov_b32_e32 v3, s6
-; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v1, s0
+; VI-NEXT: v_mov_b32_e32 v2, s1
+; VI-NEXT: v_mov_b32_e32 v3, s2
+; VI-NEXT: v_mov_b32_e32 v4, s3
; VI-NEXT: flat_store_dword v[1:2], v0
-; VI-NEXT: flat_store_byte v[3:4], v5
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; VI-NEXT: flat_store_byte v[3:4], v0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_uaddo_clamp_bit:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: s_cmp_eq_u32 s0, s1
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-NEXT: s_cmp_eq_u32 s6, s7
+; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, s6, v0
; GFX9-NEXT: s_cbranch_scc1 .LBB8_2
; GFX9-NEXT: ; %bb.1: ; %if
-; GFX9-NEXT: s_xor_b64 s[0:1], vcc, -1
+; GFX9-NEXT: s_xor_b64 s[4:5], vcc, -1
; GFX9-NEXT: .LBB8_2: ; %exit
-; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dword v1, v0, s[8:9]
-; GFX9-NEXT: global_store_byte v1, v2, s[10:11]
+; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX9-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: s_uaddo_clamp_bit:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x34
-; GFX10-NEXT: s_mov_b32 s0, 0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_add_co_u32 v0, s1, s2, s3
-; GFX10-NEXT: s_cmp_eq_u32 s2, s3
+; GFX10-NEXT: v_add_co_u32 v0, s5, s6, s7
+; GFX10-NEXT: s_cmp_eq_u32 s6, s7
; GFX10-NEXT: s_cbranch_scc1 .LBB8_2
; GFX10-NEXT: ; %bb.1: ; %if
-; GFX10-NEXT: s_xor_b32 s0, s1, -1
+; GFX10-NEXT: s_xor_b32 s4, s5, -1
; GFX10-NEXT: .LBB8_2: ; %exit
-; GFX10-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_store_dword v1, v0, s[8:9]
-; GFX10-NEXT: global_store_byte v1, v2, s[10:11]
+; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s4
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_byte v1, v2, s[2:3]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_uaddo_clamp_bit:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x34
-; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_add_co_u32 v0, s1, s2, s3
-; GFX11-NEXT: s_cmp_eq_u32 s2, s3
+; GFX11-NEXT: v_add_co_u32 v0, s5, s6, s7
+; GFX11-NEXT: s_cmp_eq_u32 s6, s7
; GFX11-NEXT: s_cbranch_scc1 .LBB8_2
; GFX11-NEXT: ; %bb.1: ; %if
-; GFX11-NEXT: s_xor_b32 s0, s1, -1
+; GFX11-NEXT: s_xor_b32 s4, s5, -1
; GFX11-NEXT: .LBB8_2: ; %exit
-; GFX11-NEXT: s_load_b128 s[4:7], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1, s4
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b32 v1, v0, s[4:5]
-; GFX11-NEXT: global_store_b8 v1, v2, s[6:7]
+; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-NEXT: global_store_b8 v1, v2, s[2:3]
; GFX11-NEXT: s_endpgm
entry:
%uadd = call { i32, i1 } @llvm.uadd.with.overflow.i32(i32 %a, i32 %b)
diff --git a/llvm/test/CodeGen/AMDGPU/udiv.ll b/llvm/test/CodeGen/AMDGPU/udiv.ll
index 50b863f974138..af432f83cb22d 100644
--- a/llvm/test/CodeGen/AMDGPU/udiv.ll
+++ b/llvm/test/CodeGen/AMDGPU/udiv.ll
@@ -189,65 +189,67 @@ define amdgpu_kernel void @s_udiv_i32(ptr addrspace(1) %out, i32 %a, i32 %b) {
; SI-LABEL: s_udiv_i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: v_cvt_f32_u32_e32 v0, s5
-; SI-NEXT: s_sub_i32 s2, 0, s5
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: v_cvt_f32_u32_e32 v0, s3
+; SI-NEXT: s_sub_i32 s4, 0, s3
+; SI-NEXT: s_mov_b32 s5, s1
; SI-NEXT: v_rcp_f32_e32 v0, v0
; SI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; SI-NEXT: v_cvt_u32_f32_e32 v0, v0
-; SI-NEXT: v_mul_lo_u32 v1, s2, v0
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_mul_lo_u32 v1, s4, v0
+; SI-NEXT: s_mov_b32 s4, s0
; SI-NEXT: v_mul_hi_u32 v1, v0, v1
; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; SI-NEXT: v_mul_hi_u32 v0, s4, v0
-; SI-NEXT: v_readfirstlane_b32 s6, v0
-; SI-NEXT: s_mul_i32 s6, s6, s5
-; SI-NEXT: s_sub_i32 s4, s4, s6
-; SI-NEXT: s_sub_i32 s6, s4, s5
+; SI-NEXT: v_mul_hi_u32 v0, s2, v0
+; SI-NEXT: v_readfirstlane_b32 s0, v0
+; SI-NEXT: s_mul_i32 s0, s0, s3
+; SI-NEXT: s_sub_i32 s0, s2, s0
+; SI-NEXT: s_sub_i32 s1, s0, s3
; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; SI-NEXT: s_cmp_ge_u32 s4, s5
+; SI-NEXT: s_cmp_ge_u32 s0, s3
; SI-NEXT: s_cselect_b64 vcc, -1, 0
; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT: s_cselect_b32 s4, s6, s4
+; SI-NEXT: s_cselect_b32 s0, s1, s0
; SI-NEXT: v_add_i32_e32 v1, vcc, 1, v0
-; SI-NEXT: s_cmp_ge_u32 s4, s5
+; SI-NEXT: s_cmp_ge_u32 s0, s3
; SI-NEXT: s_cselect_b64 vcc, -1, 0
; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_udiv_i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; VI-NEXT: v_cvt_f32_u32_e32 v0, s5
-; VI-NEXT: s_sub_i32 s2, 0, s5
-; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: v_cvt_f32_u32_e32 v0, s3
+; VI-NEXT: s_sub_i32 s4, 0, s3
+; VI-NEXT: s_mov_b32 s5, s1
; VI-NEXT: v_rcp_f32_e32 v0, v0
; VI-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; VI-NEXT: v_cvt_u32_f32_e32 v0, v0
-; VI-NEXT: v_mul_lo_u32 v1, s2, v0
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_mul_lo_u32 v1, s4, v0
+; VI-NEXT: s_mov_b32 s4, s0
; VI-NEXT: v_mul_hi_u32 v1, v0, v1
; VI-NEXT: v_add_u32_e32 v0, vcc, v0, v1
-; VI-NEXT: v_mul_hi_u32 v0, s4, v0
-; VI-NEXT: v_readfirstlane_b32 s6, v0
-; VI-NEXT: s_mul_i32 s6, s6, s5
-; VI-NEXT: s_sub_i32 s4, s4, s6
-; VI-NEXT: s_sub_i32 s6, s4, s5
+; VI-NEXT: v_mul_hi_u32 v0, s2, v0
+; VI-NEXT: v_readfirstlane_b32 s0, v0
+; VI-NEXT: s_mul_i32 s0, s0, s3
+; VI-NEXT: s_sub_i32 s0, s2, s0
+; VI-NEXT: s_sub_i32 s1, s0, s3
; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0
-; VI-NEXT: s_cmp_ge_u32 s4, s5
+; VI-NEXT: s_cmp_ge_u32 s0, s3
; VI-NEXT: s_cselect_b64 vcc, -1, 0
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; VI-NEXT: s_cselect_b32 s4, s6, s4
+; VI-NEXT: s_cselect_b32 s0, s1, s0
; VI-NEXT: v_add_u32_e32 v1, vcc, 1, v0
-; VI-NEXT: s_cmp_ge_u32 s4, s5
+; VI-NEXT: s_cmp_ge_u32 s0, s3
; VI-NEXT: s_cselect_b64 vcc, -1, 0
; VI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
;
; GCN-LABEL: s_udiv_i32:
@@ -265,22 +267,22 @@ define amdgpu_kernel void @s_udiv_i32(ptr addrspace(1) %out, i32 %a, i32 %b) {
; GCN-NEXT: v_mul_lo_u32 v1, s4, v0
; GCN-NEXT: v_mul_hi_u32 v1, v0, v1
; GCN-NEXT: v_add_u32_e32 v0, vcc, v0, v1
-; GCN-NEXT: v_mul_hi_u32 v0, s2, v0
-; GCN-NEXT: v_readfirstlane_b32 s4, v0
-; GCN-NEXT: s_mul_i32 s4, s4, s3
-; GCN-NEXT: s_sub_i32 s2, s2, s4
-; GCN-NEXT: s_sub_i32 s4, s2, s3
-; GCN-NEXT: v_add_u32_e32 v1, vcc, 1, v0
-; GCN-NEXT: s_cmp_ge_u32 s2, s3
-; GCN-NEXT: s_cselect_b64 vcc, -1, 0
-; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; GCN-NEXT: s_cselect_b32 s2, s4, s2
-; GCN-NEXT: v_add_u32_e32 v1, vcc, 1, v0
-; GCN-NEXT: s_cmp_ge_u32 s2, s3
-; GCN-NEXT: s_cselect_b64 vcc, -1, 0
-; GCN-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
+; GCN-NEXT: v_mul_hi_u32 v2, s2, v0
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_readfirstlane_b32 s0, v2
+; GCN-NEXT: s_mul_i32 s0, s0, s3
+; GCN-NEXT: s_sub_i32 s0, s2, s0
+; GCN-NEXT: s_sub_i32 s1, s0, s3
+; GCN-NEXT: v_add_u32_e32 v3, vcc, 1, v2
+; GCN-NEXT: s_cmp_ge_u32 s0, s3
+; GCN-NEXT: s_cselect_b64 vcc, -1, 0
+; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
+; GCN-NEXT: s_cselect_b32 s0, s1, s0
+; GCN-NEXT: v_add_u32_e32 v3, vcc, 1, v2
+; GCN-NEXT: s_cmp_ge_u32 s0, s3
+; GCN-NEXT: s_cselect_b64 vcc, -1, 0
+; GCN-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
; GCN-NEXT: flat_store_dword v[0:1], v2
; GCN-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/udivrem.ll b/llvm/test/CodeGen/AMDGPU/udivrem.ll
index c6aec64d1692e..bf13dc1b46409 100644
--- a/llvm/test/CodeGen/AMDGPU/udivrem.ll
+++ b/llvm/test/CodeGen/AMDGPU/udivrem.ll
@@ -164,98 +164,102 @@ define amdgpu_kernel void @test_udivrem_v2(ptr addrspace(1) %out, <2 x i32> %x,
;
; GFX6-LABEL: test_udivrem_v2:
; GFX6: ; %bb.0:
-; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0xb
-; GFX6-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
+; GFX6-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
; GFX6-NEXT: s_mov_b32 s7, 0xf000
+; GFX6-NEXT: s_mov_b32 s6, -1
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX6-NEXT: s_sub_i32 s6, 0, s2
-; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s3
+; GFX6-NEXT: v_cvt_f32_u32_e32 v0, s8
+; GFX6-NEXT: s_sub_i32 s0, 0, s8
+; GFX6-NEXT: v_cvt_f32_u32_e32 v2, s9
; GFX6-NEXT: v_rcp_f32_e32 v0, v0
; GFX6-NEXT: v_rcp_f32_e32 v2, v2
; GFX6-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX6-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX6-NEXT: v_mul_lo_u32 v1, s6, v0
+; GFX6-NEXT: v_mul_lo_u32 v1, s0, v0
+; GFX6-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; GFX6-NEXT: v_mul_hi_u32 v1, v0, v1
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 s5, s1
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; GFX6-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX6-NEXT: v_mul_hi_u32 v0, s2, v0
; GFX6-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v2
; GFX6-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX6-NEXT: v_readfirstlane_b32 s6, v0
-; GFX6-NEXT: s_mul_i32 s6, s6, s2
-; GFX6-NEXT: s_sub_i32 s0, s0, s6
-; GFX6-NEXT: s_sub_i32 s6, s0, s2
-; GFX6-NEXT: s_cmp_ge_u32 s0, s2
-; GFX6-NEXT: s_cselect_b32 s0, s6, s0
-; GFX6-NEXT: s_sub_i32 s6, s0, s2
-; GFX6-NEXT: s_cmp_ge_u32 s0, s2
-; GFX6-NEXT: s_cselect_b32 s0, s6, s0
-; GFX6-NEXT: s_sub_i32 s2, 0, s3
-; GFX6-NEXT: v_mul_lo_u32 v0, s2, v1
-; GFX6-NEXT: s_mov_b32 s6, -1
+; GFX6-NEXT: v_readfirstlane_b32 s4, v0
+; GFX6-NEXT: s_mul_i32 s4, s4, s8
+; GFX6-NEXT: s_sub_i32 s2, s2, s4
+; GFX6-NEXT: s_sub_i32 s4, s2, s8
+; GFX6-NEXT: s_cmp_ge_u32 s2, s8
+; GFX6-NEXT: s_cselect_b32 s2, s4, s2
+; GFX6-NEXT: s_sub_i32 s4, s2, s8
+; GFX6-NEXT: s_cmp_ge_u32 s2, s8
+; GFX6-NEXT: s_cselect_b32 s2, s4, s2
+; GFX6-NEXT: s_sub_i32 s4, 0, s9
+; GFX6-NEXT: v_mul_lo_u32 v0, s4, v1
+; GFX6-NEXT: s_mov_b32 s4, s0
; GFX6-NEXT: v_mul_hi_u32 v0, v1, v0
; GFX6-NEXT: v_add_i32_e32 v0, vcc, v1, v0
-; GFX6-NEXT: v_mul_hi_u32 v0, s1, v0
-; GFX6-NEXT: v_readfirstlane_b32 s2, v0
-; GFX6-NEXT: s_mul_i32 s2, s2, s3
-; GFX6-NEXT: s_sub_i32 s1, s1, s2
-; GFX6-NEXT: s_sub_i32 s2, s1, s3
-; GFX6-NEXT: s_cmp_ge_u32 s1, s3
-; GFX6-NEXT: s_cselect_b32 s1, s2, s1
-; GFX6-NEXT: s_sub_i32 s2, s1, s3
-; GFX6-NEXT: s_cmp_ge_u32 s1, s3
-; GFX6-NEXT: s_cselect_b32 s1, s2, s1
-; GFX6-NEXT: v_mov_b32_e32 v0, s0
-; GFX6-NEXT: v_mov_b32_e32 v1, s1
+; GFX6-NEXT: v_mul_hi_u32 v0, s3, v0
+; GFX6-NEXT: v_readfirstlane_b32 s0, v0
+; GFX6-NEXT: s_mul_i32 s0, s0, s9
+; GFX6-NEXT: s_sub_i32 s0, s3, s0
+; GFX6-NEXT: s_sub_i32 s1, s0, s9
+; GFX6-NEXT: s_cmp_ge_u32 s0, s9
+; GFX6-NEXT: s_cselect_b32 s0, s1, s0
+; GFX6-NEXT: s_sub_i32 s1, s0, s9
+; GFX6-NEXT: s_cmp_ge_u32 s0, s9
+; GFX6-NEXT: s_cselect_b32 s0, s1, s0
+; GFX6-NEXT: v_mov_b32_e32 v0, s2
+; GFX6-NEXT: v_mov_b32_e32 v1, s0
; GFX6-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: test_udivrem_v2:
; GFX8: ; %bb.0:
-; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX8-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
+; GFX8-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
-; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s2
-; GFX8-NEXT: s_sub_i32 s6, 0, s2
-; GFX8-NEXT: v_cvt_f32_u32_e32 v2, s3
+; GFX8-NEXT: v_cvt_f32_u32_e32 v0, s6
+; GFX8-NEXT: s_sub_i32 s0, 0, s6
+; GFX8-NEXT: v_cvt_f32_u32_e32 v2, s7
; GFX8-NEXT: v_rcp_f32_e32 v0, v0
; GFX8-NEXT: v_rcp_f32_e32 v2, v2
; GFX8-NEXT: v_mul_f32_e32 v0, 0x4f7ffffe, v0
; GFX8-NEXT: v_cvt_u32_f32_e32 v0, v0
-; GFX8-NEXT: v_mul_lo_u32 v1, s6, v0
+; GFX8-NEXT: v_mul_lo_u32 v1, s0, v0
+; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: v_mul_hi_u32 v1, v0, v1
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1
-; GFX8-NEXT: v_mul_hi_u32 v0, s0, v0
+; GFX8-NEXT: s_waitcnt lgkmcnt(0)
+; GFX8-NEXT: v_mul_hi_u32 v0, s2, v0
; GFX8-NEXT: v_mul_f32_e32 v1, 0x4f7ffffe, v2
; GFX8-NEXT: v_cvt_u32_f32_e32 v1, v1
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
-; GFX8-NEXT: v_readfirstlane_b32 s6, v0
-; GFX8-NEXT: s_mul_i32 s6, s6, s2
-; GFX8-NEXT: s_sub_i32 s0, s0, s6
-; GFX8-NEXT: s_sub_i32 s6, s0, s2
-; GFX8-NEXT: s_cmp_ge_u32 s0, s2
-; GFX8-NEXT: s_cselect_b32 s0, s6, s0
-; GFX8-NEXT: s_sub_i32 s6, s0, s2
-; GFX8-NEXT: s_cmp_ge_u32 s0, s2
-; GFX8-NEXT: s_cselect_b32 s0, s6, s0
-; GFX8-NEXT: s_sub_i32 s2, 0, s3
-; GFX8-NEXT: v_mul_lo_u32 v0, s2, v1
-; GFX8-NEXT: v_mov_b32_e32 v3, s5
+; GFX8-NEXT: v_readfirstlane_b32 s4, v0
+; GFX8-NEXT: s_mul_i32 s4, s4, s6
+; GFX8-NEXT: s_sub_i32 s2, s2, s4
+; GFX8-NEXT: s_sub_i32 s4, s2, s6
+; GFX8-NEXT: s_cmp_ge_u32 s2, s6
+; GFX8-NEXT: s_cselect_b32 s2, s4, s2
+; GFX8-NEXT: s_sub_i32 s4, s2, s6
+; GFX8-NEXT: s_cmp_ge_u32 s2, s6
+; GFX8-NEXT: s_cselect_b32 s2, s4, s2
+; GFX8-NEXT: s_sub_i32 s4, 0, s7
+; GFX8-NEXT: v_mul_lo_u32 v0, s4, v1
; GFX8-NEXT: v_mul_hi_u32 v0, v1, v0
; GFX8-NEXT: v_add_u32_e32 v0, vcc, v1, v0
-; GFX8-NEXT: v_mul_hi_u32 v1, s1, v0
+; GFX8-NEXT: v_mul_hi_u32 v2, s3, v0
; GFX8-NEXT: v_mov_b32_e32 v0, s0
-; GFX8-NEXT: v_readfirstlane_b32 s0, v1
-; GFX8-NEXT: s_mul_i32 s0, s0, s3
-; GFX8-NEXT: s_sub_i32 s0, s1, s0
-; GFX8-NEXT: s_sub_i32 s1, s0, s3
-; GFX8-NEXT: s_cmp_ge_u32 s0, s3
+; GFX8-NEXT: v_mov_b32_e32 v1, s1
+; GFX8-NEXT: v_readfirstlane_b32 s0, v2
+; GFX8-NEXT: s_mul_i32 s0, s0, s7
+; GFX8-NEXT: s_sub_i32 s0, s3, s0
+; GFX8-NEXT: s_sub_i32 s1, s0, s7
+; GFX8-NEXT: s_cmp_ge_u32 s0, s7
; GFX8-NEXT: s_cselect_b32 s0, s1, s0
-; GFX8-NEXT: s_sub_i32 s1, s0, s3
-; GFX8-NEXT: s_cmp_ge_u32 s0, s3
+; GFX8-NEXT: s_sub_i32 s1, s0, s7
+; GFX8-NEXT: s_cmp_ge_u32 s0, s7
; GFX8-NEXT: s_cselect_b32 s0, s1, s0
-; GFX8-NEXT: v_mov_b32_e32 v1, s0
-; GFX8-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_mov_b32_e32 v3, s0
+; GFX8-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
; GFX8-NEXT: s_endpgm
%result0 = udiv <2 x i32> %x, %y
store <2 x i32> %result0, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
index 983acfc2c0699..6eca8d49cf22a 100644
--- a/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
+++ b/llvm/test/CodeGen/AMDGPU/uint_to_fp.f64.ll
@@ -288,38 +288,22 @@ define amdgpu_kernel void @s_uint_to_fp_v4i64_to_v4f64(ptr addrspace(1) %out, <4
}
define amdgpu_kernel void @s_uint_to_fp_i32_to_f64(ptr addrspace(1) %out, i32 %in) {
-; SI-LABEL: s_uint_to_fp_i32_to_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[8:9], 0x2
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_add_i32 s12, s12, s17
-; SI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; SI-NEXT: v_mov_b32_e32 v3, s1
-; SI-NEXT: v_mov_b32_e32 v2, s0
-; SI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; SI-NEXT: s_endpgm
-;
-; VI-LABEL: s_uint_to_fp_i32_to_f64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: s_uint_to_fp_i32_to_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
+; GCN-NEXT: v_mov_b32_e32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: s_uint_to_fp_i32_to_f64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
@@ -340,8 +324,8 @@ define amdgpu_kernel void @s_uint_to_fp_v2i32_to_v2f64(ptr addrspace(1) %out, <2
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: v_cvt_f64_u32_e32 v[2:3], s3
; GCN-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; GCN-NEXT: v_mov_b32_e32 v5, s1
; GCN-NEXT: v_mov_b32_e32 v4, s0
+; GCN-NEXT: v_mov_b32_e32 v5, s1
; GCN-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NEXT: s_endpgm
;
@@ -427,44 +411,25 @@ define amdgpu_kernel void @s_uint_to_fp_v4i32_to_v4f64(ptr addrspace(1) %out, <4
; We can't fold the SGPRs into v_cndmask_b32_e32, because it already
; uses an SGPR (implicit vcc).
define amdgpu_kernel void @uint_to_fp_i1_to_f64(ptr addrspace(1) %out, i32 %in) {
-; SI-LABEL: uint_to_fp_i1_to_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[8:9], 0x2
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_add_i32 s12, s12, s17
-; SI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_eq_u32 s2, 0
-; SI-NEXT: s_cselect_b32 s2, 0x3ff00000, 0
-; SI-NEXT: v_mov_b32_e32 v3, s1
-; SI-NEXT: v_mov_b32_e32 v0, 0
-; SI-NEXT: v_mov_b32_e32 v1, s2
-; SI-NEXT: v_mov_b32_e32 v2, s0
-; SI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; SI-NEXT: s_endpgm
-;
-; VI-LABEL: uint_to_fp_i1_to_f64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s2, 0
-; VI-NEXT: s_cselect_b32 s2, 0x3ff00000, 0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: uint_to_fp_i1_to_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: uint_to_fp_i1_to_f64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_eq_u32 s2, 0
@@ -479,44 +444,25 @@ define amdgpu_kernel void @uint_to_fp_i1_to_f64(ptr addrspace(1) %out, i32 %in)
}
define amdgpu_kernel void @uint_to_fp_i1_to_f64_load(ptr addrspace(1) %out, i1 %in) {
-; SI-LABEL: uint_to_fp_i1_to_f64_load:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[8:9], 0x2
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_add_i32 s12, s12, s17
-; SI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_bitcmp1_b32 s2, 0
-; SI-NEXT: s_cselect_b64 s[2:3], -1, 0
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; SI-NEXT: v_cvt_f64_u32_e32 v[0:1], v0
-; SI-NEXT: v_mov_b32_e32 v3, s1
-; SI-NEXT: v_mov_b32_e32 v2, s0
-; SI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; SI-NEXT: s_endpgm
-;
-; VI-LABEL: uint_to_fp_i1_to_f64_load:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_bitcmp1_b32 s2, 0
-; VI-NEXT: s_cselect_b64 s[2:3], -1, 0
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
-; VI-NEXT: v_cvt_f64_u32_e32 v[0:1], v0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: uint_to_fp_i1_to_f64_load:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_bitcmp1_b32 s2, 0
+; GCN-NEXT: s_cselect_b64 s[2:3], -1, 0
+; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
+; GCN-NEXT: v_cvt_f64_u32_e32 v[0:1], v0
+; GCN-NEXT: v_mov_b32_e32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: uint_to_fp_i1_to_f64_load:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_bitcmp1_b32 s2, 0
@@ -531,40 +477,23 @@ define amdgpu_kernel void @uint_to_fp_i1_to_f64_load(ptr addrspace(1) %out, i1 %
}
define amdgpu_kernel void @s_uint_to_fp_i8_to_f64(ptr addrspace(1) %out, i8 %in) {
-; SI-LABEL: s_uint_to_fp_i8_to_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[8:9], 0x2
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_add_i32 s12, s12, s17
-; SI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_and_b32 s2, s2, 0xff
-; SI-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; SI-NEXT: v_mov_b32_e32 v3, s1
-; SI-NEXT: v_mov_b32_e32 v2, s0
-; SI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; SI-NEXT: s_endpgm
-;
-; VI-LABEL: s_uint_to_fp_i8_to_f64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_and_b32 s2, s2, 0xff
-; VI-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: s_uint_to_fp_i8_to_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_and_b32 s2, s2, 0xff
+; GCN-NEXT: v_cvt_f64_u32_e32 v[0:1], s2
+; GCN-NEXT: v_mov_b32_e32 v2, s0
+; GCN-NEXT: v_mov_b32_e32 v3, s1
+; GCN-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: s_uint_to_fp_i8_to_f64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v2, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_and_b32 s2, s2, 0xff
@@ -605,44 +534,25 @@ define double @v_uint_to_fp_i8_to_f64(i8 %in) {
}
define amdgpu_kernel void @s_select_uint_to_fp_i1_vals_f64(ptr addrspace(1) %out, i32 %in) {
-; SI-LABEL: s_select_uint_to_fp_i1_vals_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[8:9], 0x2
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_add_i32 s12, s12, s17
-; SI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_eq_u32 s2, 0
-; SI-NEXT: s_cselect_b32 s2, 0x3ff00000, 0
-; SI-NEXT: v_mov_b32_e32 v3, s1
-; SI-NEXT: v_mov_b32_e32 v0, 0
-; SI-NEXT: v_mov_b32_e32 v1, s2
-; SI-NEXT: v_mov_b32_e32 v2, s0
-; SI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; SI-NEXT: s_endpgm
-;
-; VI-LABEL: s_select_uint_to_fp_i1_vals_f64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s2, 0
-; VI-NEXT: s_cselect_b32 s2, 0x3ff00000, 0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: s_select_uint_to_fp_i1_vals_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: s_select_uint_to_fp_i1_vals_f64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_eq_u32 s2, 0
@@ -686,44 +596,25 @@ define void @v_select_uint_to_fp_i1_vals_f64(ptr addrspace(1) %out, i32 %in) {
}
define amdgpu_kernel void @s_select_uint_to_fp_i1_vals_i64(ptr addrspace(1) %out, i32 %in) {
-; SI-LABEL: s_select_uint_to_fp_i1_vals_i64:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[8:9], 0x2
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_add_i32 s12, s12, s17
-; SI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_eq_u32 s2, 0
-; SI-NEXT: s_cselect_b32 s2, 0x3ff00000, 0
-; SI-NEXT: v_mov_b32_e32 v3, s1
-; SI-NEXT: v_mov_b32_e32 v0, 0
-; SI-NEXT: v_mov_b32_e32 v1, s2
-; SI-NEXT: v_mov_b32_e32 v2, s0
-; SI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; SI-NEXT: s_endpgm
-;
-; VI-LABEL: s_select_uint_to_fp_i1_vals_i64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s2, 0
-; VI-NEXT: s_cselect_b32 s2, 0x3ff00000, 0
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: s_select_uint_to_fp_i1_vals_i64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, 0x3ff00000, 0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: s_select_uint_to_fp_i1_vals_i64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_eq_u32 s2, 0
@@ -768,44 +659,25 @@ define void @v_select_uint_to_fp_i1_vals_i64(ptr addrspace(1) %out, i32 %in) {
; TODO: This should swap the selected order / invert the compare and do it.
define amdgpu_kernel void @s_swap_select_uint_to_fp_i1_vals_f64(ptr addrspace(1) %out, i32 %in) {
-; SI-LABEL: s_swap_select_uint_to_fp_i1_vals_f64:
-; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s2, s[8:9], 0x2
-; SI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; SI-NEXT: s_add_i32 s12, s12, s17
-; SI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; SI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_eq_u32 s2, 0
-; SI-NEXT: s_cselect_b32 s2, 0, 0x3ff00000
-; SI-NEXT: v_mov_b32_e32 v3, s1
-; SI-NEXT: v_mov_b32_e32 v0, 0
-; SI-NEXT: v_mov_b32_e32 v1, s2
-; SI-NEXT: v_mov_b32_e32 v2, s0
-; SI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; SI-NEXT: s_endpgm
-;
-; VI-LABEL: s_swap_select_uint_to_fp_i1_vals_f64:
-; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[8:9], 0x8
-; VI-NEXT: s_load_dwordx2 s[0:1], s[8:9], 0x0
-; VI-NEXT: s_add_i32 s12, s12, s17
-; VI-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; VI-NEXT: s_mov_b32 flat_scratch_lo, s13
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_eq_u32 s2, 0
-; VI-NEXT: s_cselect_b32 s2, 0, 0x3ff00000
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: v_mov_b32_e32 v1, s2
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
-; VI-NEXT: s_endpgm
+; GCN-LABEL: s_swap_select_uint_to_fp_i1_vals_f64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[8:9], 0x0
+; GCN-NEXT: s_add_i32 s12, s12, s17
+; GCN-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
+; GCN-NEXT: s_mov_b32 flat_scratch_lo, s13
+; GCN-NEXT: v_mov_b32_e32 v2, 0
+; GCN-NEXT: s_waitcnt lgkmcnt(0)
+; GCN-NEXT: s_cmp_eq_u32 s2, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: s_cselect_b32 s0, 0, 0x3ff00000
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: v_mov_b32_e32 v3, s0
+; GCN-NEXT: flat_store_dwordx2 v[0:1], v[2:3]
+; GCN-NEXT: s_endpgm
;
; GFX942-LABEL: s_swap_select_uint_to_fp_i1_vals_f64:
; GFX942: ; %bb.0:
-; GFX942-NEXT: s_load_dword s2, s[4:5], 0x8
-; GFX942-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x0
; GFX942-NEXT: v_mov_b32_e32 v0, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: s_cmp_eq_u32 s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/uniform-cfg.ll b/llvm/test/CodeGen/AMDGPU/uniform-cfg.ll
index d6495648a594f..6d92d8f185726 100644
--- a/llvm/test/CodeGen/AMDGPU/uniform-cfg.ll
+++ b/llvm/test/CodeGen/AMDGPU/uniform-cfg.ll
@@ -219,36 +219,32 @@ done:
define amdgpu_kernel void @uniform_if_move_valu(ptr addrspace(1) %out, float %a) {
; SI-LABEL: uniform_if_move_valu:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: v_mov_b32_e32 v0, 0x41200000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e32 v0, s0, v0
+; SI-NEXT: v_add_f32_e32 v0, s2, v0
; SI-NEXT: v_cmp_ne_u32_e32 vcc, 5, v0
; SI-NEXT: s_cbranch_vccnz .LBB4_2
; SI-NEXT: ; %bb.1: ; %if
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: .LBB4_2: ; %endif
; SI-NEXT: s_endpgm
;
; VI-LABEL: uniform_if_move_valu:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s0, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 0x41200000
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e32 v0, s0, v0
+; VI-NEXT: v_add_f32_e32 v0, s2, v0
; VI-NEXT: v_cmp_ne_u32_e32 vcc, 5, v0
; VI-NEXT: s_cbranch_vccnz .LBB4_2
; VI-NEXT: ; %bb.1: ; %if
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: .LBB4_2: ; %endif
; VI-NEXT: s_endpgm
@@ -271,36 +267,32 @@ endif:
define amdgpu_kernel void @uniform_if_move_valu_commute(ptr addrspace(1) %out, float %a) {
; SI-LABEL: uniform_if_move_valu_commute:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: v_mov_b32_e32 v0, 0x41200000
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_add_f32_e32 v0, s0, v0
+; SI-NEXT: v_add_f32_e32 v0, s2, v0
; SI-NEXT: v_cmp_gt_u32_e32 vcc, 6, v0
; SI-NEXT: s_cbranch_vccnz .LBB5_2
; SI-NEXT: ; %bb.1: ; %if
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v0, 0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: .LBB5_2: ; %endif
; SI-NEXT: s_endpgm
;
; VI-LABEL: uniform_if_move_valu_commute:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s0, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_mov_b32_e32 v0, 0x41200000
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_add_f32_e32 v0, s0, v0
+; VI-NEXT: v_add_f32_e32 v0, s2, v0
; VI-NEXT: v_cmp_gt_u32_e32 vcc, 6, v0
; VI-NEXT: s_cbranch_vccnz .LBB5_2
; VI-NEXT: ; %bb.1: ; %if
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: v_mov_b32_e32 v0, 0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: .LBB5_2: ; %endif
; VI-NEXT: s_endpgm
@@ -322,10 +314,9 @@ endif:
define amdgpu_kernel void @uniform_if_else_ret(ptr addrspace(1) nocapture %out, i32 %a) {
; SI-LABEL: uniform_if_else_ret:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
+; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_cmp_lg_u32 s2, 0
; SI-NEXT: s_cbranch_scc0 .LBB6_2
; SI-NEXT: ; %bb.1: ; %if.else
@@ -341,10 +332,9 @@ define amdgpu_kernel void @uniform_if_else_ret(ptr addrspace(1) nocapture %out,
;
; VI-LABEL: uniform_if_else_ret:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_cmp_lg_u32 s2, 0
; VI-NEXT: s_cbranch_scc0 .LBB6_2
; VI-NEXT: ; %bb.1: ; %if.else
@@ -448,37 +438,33 @@ if.end: ; preds = %if.else, %if.then
define amdgpu_kernel void @icmp_2_users(ptr addrspace(1) %out, i32 %cond) {
; SI-LABEL: icmp_2_users:
; SI: ; %bb.0: ; %main_body
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_cmp_gt_i32 s2, 0
-; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
; SI-NEXT: s_cmp_lt_i32 s2, 1
; SI-NEXT: s_cbranch_scc1 .LBB8_2
; SI-NEXT: ; %bb.1: ; %IF
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
+; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; SI-NEXT: .LBB8_2: ; %ENDIF
; SI-NEXT: s_endpgm
;
; VI-LABEL: icmp_2_users:
; VI: ; %bb.0: ; %main_body
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_cmp_gt_i32 s2, 0
-; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: s_cselect_b64 s[4:5], -1, 0
; VI-NEXT: s_cmp_lt_i32 s2, 1
; VI-NEXT: s_cbranch_scc1 .LBB8_2
; VI-NEXT: ; %bb.1: ; %IF
-; VI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s7, 0xf000
-; VI-NEXT: s_mov_b32 s6, -1
-; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[0:1]
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
+; VI-NEXT: s_mov_b32 s3, 0xf000
+; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, -1, s[4:5]
+; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: .LBB8_2: ; %ENDIF
; VI-NEXT: s_endpgm
main_body:
@@ -497,7 +483,7 @@ ENDIF: ; preds = %IF, %main_body
define amdgpu_kernel void @icmp_users_different_blocks(i32 %cond0, i32 %cond1, ptr addrspace(1) %out) {
; SI-LABEL: icmp_users_different_blocks:
; SI: ; %bb.0: ; %bb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_cmp_lt_i32 s0, 1
; SI-NEXT: s_cbranch_scc1 .LBB9_2
@@ -510,17 +496,17 @@ define amdgpu_kernel void @icmp_users_different_blocks(i32 %cond0, i32 %cond1, p
; SI-NEXT: s_endpgm
; SI-NEXT: .LBB9_3: ; %bb7
; SI-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xb
; SI-NEXT: v_sub_i32_e32 v0, vcc, v0, v1
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
+; SI-NEXT: s_mov_b32 s4, s2
+; SI-NEXT: s_mov_b32 s5, s3
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: icmp_users_different_blocks:
; VI: ; %bb.0: ; %bb
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_cmp_lt_i32 s0, 1
; VI-NEXT: s_cbranch_scc1 .LBB9_2
@@ -533,12 +519,12 @@ define amdgpu_kernel void @icmp_users_different_blocks(i32 %cond0, i32 %cond1, p
; VI-NEXT: s_endpgm
; VI-NEXT: .LBB9_3: ; %bb7
; VI-NEXT: v_cndmask_b32_e64 v1, 0, 1, s[0:1]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x2c
; VI-NEXT: v_sub_u32_e32 v0, vcc, v0, v1
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
+; VI-NEXT: s_mov_b32 s4, s2
+; VI-NEXT: s_mov_b32 s5, s3
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_endpgm
bb:
%tmp = tail call i32 @llvm.amdgcn.workitem.id.x() #0
@@ -605,18 +591,21 @@ define amdgpu_kernel void @uniform_inside_divergent(ptr addrspace(1) %out, i32 %
; SI-NEXT: s_and_saveexec_b64 s[0:1], vcc
; SI-NEXT: s_cbranch_execz .LBB11_2
; SI-NEXT: ; %bb.1: ; %if
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: v_mov_b32_e32 v0, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_lg_u32 s6, 0
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: s_cmp_lg_u32 s2, 0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_cbranch_scc0 .LBB11_3
; SI-NEXT: .LBB11_2: ; %endif
; SI-NEXT: s_endpgm
; SI-NEXT: .LBB11_3: ; %if_uniform
+; SI-NEXT: s_mov_b32 s2, s6
+; SI-NEXT: s_mov_b32 s3, s7
; SI-NEXT: s_waitcnt expcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, 1
; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
@@ -628,18 +617,21 @@ define amdgpu_kernel void @uniform_inside_divergent(ptr addrspace(1) %out, i32 %
; VI-NEXT: s_and_saveexec_b64 s[0:1], vcc
; VI-NEXT: s_cbranch_execz .LBB11_2
; VI-NEXT: ; %bb.1: ; %if
-; VI-NEXT: s_load_dword s6, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b32 s7, 0xf000
+; VI-NEXT: s_mov_b32 s6, -1
; VI-NEXT: v_mov_b32_e32 v0, 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_lg_u32 s6, 0
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s4, s0
+; VI-NEXT: s_mov_b32 s5, s1
+; VI-NEXT: s_cmp_lg_u32 s2, 0
+; VI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; VI-NEXT: s_cbranch_scc0 .LBB11_3
; VI-NEXT: .LBB11_2: ; %endif
; VI-NEXT: s_endpgm
; VI-NEXT: .LBB11_3: ; %if_uniform
+; VI-NEXT: s_mov_b32 s2, s6
+; VI-NEXT: s_mov_b32 s3, s7
; VI-NEXT: v_mov_b32_e32 v0, 1
; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
; VI-NEXT: s_endpgm
@@ -664,19 +656,17 @@ endif:
define amdgpu_kernel void @divergent_inside_uniform(ptr addrspace(1) %out, i32 %cond) {
; SI-LABEL: divergent_inside_uniform:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dword s0, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_cmp_lg_u32 s0, 0
+; SI-NEXT: s_cmp_lg_u32 s2, 0
; SI-NEXT: s_cbranch_scc0 .LBB12_2
; SI-NEXT: .LBB12_1: ; %endif
; SI-NEXT: s_endpgm
; SI-NEXT: .LBB12_2: ; %if
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, -1
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0
-; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_store_dword v1, off, s[0:3], 0
; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_cbranch_execz .LBB12_1
@@ -687,19 +677,17 @@ define amdgpu_kernel void @divergent_inside_uniform(ptr addrspace(1) %out, i32 %
;
; VI-LABEL: divergent_inside_uniform:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dword s0, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_cmp_lg_u32 s0, 0
+; VI-NEXT: s_cmp_lg_u32 s2, 0
; VI-NEXT: s_cbranch_scc0 .LBB12_2
; VI-NEXT: .LBB12_1: ; %endif
; VI-NEXT: s_endpgm
; VI-NEXT: .LBB12_2: ; %if
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NEXT: s_mov_b32 s3, 0xf000
; VI-NEXT: s_mov_b32 s2, -1
; VI-NEXT: v_mov_b32_e32 v1, 0
; VI-NEXT: v_cmp_gt_u32_e32 vcc, 16, v0
-; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: buffer_store_dword v1, off, s[0:3], 0
; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_cbranch_execz .LBB12_1
@@ -728,19 +716,20 @@ endif:
define amdgpu_kernel void @divergent_if_uniform_if(ptr addrspace(1) %out, i32 %cond) {
; SI-LABEL: divergent_if_uniform_if:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; SI-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; SI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; SI-NEXT: s_cbranch_execz .LBB13_2
; SI-NEXT: ; %bb.1: ; %if
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, 1
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s10, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: v_mov_b32_e32 v0, 1
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SI-NEXT: .LBB13_2: ; %endif
-; SI-NEXT: s_or_b64 exec, exec, s[6:7]
-; SI-NEXT: s_load_dword s2, s[4:5], 0xb
+; SI-NEXT: s_or_b64 exec, exec, s[4:5]
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_cmp_lg_u32 s2, 0
; SI-NEXT: s_cbranch_scc0 .LBB13_4
@@ -756,19 +745,20 @@ define amdgpu_kernel void @divergent_if_uniform_if(ptr addrspace(1) %out, i32 %c
;
; VI-LABEL: divergent_if_uniform_if:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
-; VI-NEXT: s_and_saveexec_b64 s[6:7], vcc
+; VI-NEXT: s_and_saveexec_b64 s[4:5], vcc
; VI-NEXT: s_cbranch_execz .LBB13_2
; VI-NEXT: ; %bb.1: ; %if
-; VI-NEXT: s_mov_b32 s3, 0xf000
-; VI-NEXT: s_mov_b32 s2, -1
-; VI-NEXT: v_mov_b32_e32 v0, 1
+; VI-NEXT: s_mov_b32 s11, 0xf000
+; VI-NEXT: s_mov_b32 s10, -1
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; VI-NEXT: s_mov_b32 s8, s0
+; VI-NEXT: s_mov_b32 s9, s1
+; VI-NEXT: v_mov_b32_e32 v0, 1
+; VI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; VI-NEXT: .LBB13_2: ; %endif
-; VI-NEXT: s_or_b64 exec, exec, s[6:7]
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_or_b64 exec, exec, s[4:5]
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s2, 0
; VI-NEXT: s_cbranch_scc0 .LBB13_4
diff --git a/llvm/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll b/llvm/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll
index f50576eaad0cf..ba95bf2e87e65 100644
--- a/llvm/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll
+++ b/llvm/test/CodeGen/AMDGPU/use-sgpr-multiple-times.ll
@@ -8,7 +8,8 @@ declare float @llvm.amdgcn.div.fixup.f32(float, float, float) #1
; GCN-LABEL: {{^}}test_sgpr_use_twice_binop:
-; GCN: s_load_dword [[SGPR:s[0-9]+]],
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN: s_mov_b32 [[SGPR:s[0-9]+]], s{{[0-9]+}}
; GCN: v_add_f32_e64 [[RESULT:v[0-9]+]], [[SGPR]], [[SGPR]]
; GCN: buffer_store_dword [[RESULT]]
define amdgpu_kernel void @test_sgpr_use_twice_binop(ptr addrspace(1) %out, float %a) #0 {
@@ -18,7 +19,8 @@ define amdgpu_kernel void @test_sgpr_use_twice_binop(ptr addrspace(1) %out, floa
}
; GCN-LABEL: {{^}}test_sgpr_use_three_ternary_op:
-; GCN: s_load_dword [[SGPR:s[0-9]+]],
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN: s_mov_b32 [[SGPR:s[0-9]+]], s{{[0-9]+}}
; GCN: v_fma_f32 [[RESULT:v[0-9]+]], [[SGPR]], [[SGPR]], [[SGPR]]
; GCN: buffer_store_dword [[RESULT]]
define amdgpu_kernel void @test_sgpr_use_three_ternary_op(ptr addrspace(1) %out, float %a) #0 {
@@ -97,7 +99,8 @@ define amdgpu_kernel void @test_sgpr_use_twice_ternary_op_b_a_a(ptr addrspace(1)
}
; GCN-LABEL: {{^}}test_sgpr_use_twice_ternary_op_a_a_imm:
-; GCN: s_load_dword [[SGPR:s[0-9]+]]
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN: s_mov_b32 [[SGPR:s[0-9]+]], s{{[0-9]+}}
; GCN: v_fma_f32 [[RESULT:v[0-9]+]], [[SGPR]], [[SGPR]], 2.0
; GCN: buffer_store_dword [[RESULT]]
define amdgpu_kernel void @test_sgpr_use_twice_ternary_op_a_a_imm(ptr addrspace(1) %out, float %a) #0 {
@@ -107,7 +110,8 @@ define amdgpu_kernel void @test_sgpr_use_twice_ternary_op_a_a_imm(ptr addrspace(
}
; GCN-LABEL: {{^}}test_sgpr_use_twice_ternary_op_a_imm_a:
-; GCN: s_load_dword [[SGPR:s[0-9]+]]
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN: s_mov_b32 [[SGPR:s[0-9]+]], s{{[0-9]+}}
; GCN: v_fma_f32 [[RESULT:v[0-9]+]], [[SGPR]], 2.0, [[SGPR]]
; GCN: buffer_store_dword [[RESULT]]
define amdgpu_kernel void @test_sgpr_use_twice_ternary_op_a_imm_a(ptr addrspace(1) %out, float %a) #0 {
@@ -118,7 +122,8 @@ define amdgpu_kernel void @test_sgpr_use_twice_ternary_op_a_imm_a(ptr addrspace(
; Don't use fma since fma c, x, y is canonicalized to fma x, c, y
; GCN-LABEL: {{^}}test_sgpr_use_twice_ternary_op_imm_a_a:
-; GCN: s_load_dword [[SGPR:s[0-9]+]]
+; GCN: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN: s_mov_b32 [[SGPR:s[0-9]+]], s{{[0-9]+}}
; GCN: v_div_fixup_f32 [[RESULT:v[0-9]+]], 2.0, [[SGPR]], [[SGPR]]
; GCN: buffer_store_dword [[RESULT]]
define amdgpu_kernel void @test_sgpr_use_twice_ternary_op_imm_a_a(ptr addrspace(1) %out, float %a) #0 {
@@ -128,7 +133,8 @@ define amdgpu_kernel void @test_sgpr_use_twice_ternary_op_imm_a_a(ptr addrspace(
}
; GCN-LABEL: {{^}}test_sgpr_use_twice_ternary_op_a_a_kimm:
-; GCN-DAG: s_load_dword [[SGPR:s[0-9]+]]
+; GCN-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN-DAG: s_mov_b32 [[SGPR:s[0-9]+]], s{{[0-9]+}}
; GCN-DAG: v_mov_b32_e32 [[VK:v[0-9]+]], 0x44800000
; GCN: v_fma_f32 [[RESULT:v[0-9]+]], [[SGPR]], [[SGPR]], [[VK]]
; GCN: buffer_store_dword [[RESULT]]
@@ -139,8 +145,8 @@ define amdgpu_kernel void @test_sgpr_use_twice_ternary_op_a_a_kimm(ptr addrspace
}
; GCN-LABEL: {{^}}test_literal_use_twice_ternary_op_k_k_s:
-; GCN-DAG: s_load_dword [[SGPR:s[0-9]+]]
-; GCN-DAG: v_mov_b32_e32 [[VGPR:v[0-9]+]], [[SGPR]]
+; GCN-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN-DAG: v_mov_b32_e32 [[VGPR:v[0-9]+]], [[SGPR:s[0-9]+]]
; GCN-DAG: s_mov_b32 [[SK:s[0-9]+]], 0x44800000
; GCN: v_fma_f32 [[RESULT0:v[0-9]+]], [[SK]], [[SK]], [[VGPR]]
; GCN: buffer_store_dword [[RESULT0]]
@@ -169,8 +175,8 @@ define amdgpu_kernel void @test_literal_use_twice_ternary_op_k_k_s_x2(ptr addrsp
}
; GCN-LABEL: {{^}}test_literal_use_twice_ternary_op_k_s_k:
-; GCN-DAG: s_load_dword [[SGPR:s[0-9]+]]
-; GCN-DAG: v_mov_b32_e32 [[VGPR:v[0-9]+]], [[SGPR]]
+; GCN-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN-DAG: v_mov_b32_e32 [[VGPR:v[0-9]+]], [[SGPR:s[0-9]+]]
; GCN-DAG: s_mov_b32 [[SK:s[0-9]+]], 0x44800000
; GCN: v_fma_f32 [[RESULT:v[0-9]+]], [[VGPR]], [[SK]], [[SK]]
; GCN: buffer_store_dword [[RESULT]]
@@ -199,8 +205,8 @@ define amdgpu_kernel void @test_literal_use_twice_ternary_op_k_s_k_x2(ptr addrsp
}
; GCN-LABEL: {{^}}test_literal_use_twice_ternary_op_s_k_k:
-; GCN-DAG: s_load_dword [[SGPR:s[0-9]+]]
-; GCN-DAG: v_mov_b32_e32 [[VGPR:v[0-9]+]], [[SGPR]]
+; GCN-DAG: s_load_dwordx4 s{{\[[0-9]+:[0-9]+\]}}, s{{\[[0-9]+:[0-9]+\]}}, {{0x0|0x9|0x24}}
+; GCN-DAG: v_mov_b32_e32 [[VGPR:v[0-9]+]], [[SGPR:s[0-9]+]]
; GCN-DAG: s_mov_b32 [[SK:s[0-9]+]], 0x44800000
; GCN: v_fma_f32 [[RESULT:v[0-9]+]], [[VGPR]], [[SK]], [[SK]]
; GCN: buffer_store_dword [[RESULT]]
diff --git a/llvm/test/CodeGen/AMDGPU/usubo.ll b/llvm/test/CodeGen/AMDGPU/usubo.ll
index 8a54ad301f48a..bf2189d9bcdde 100644
--- a/llvm/test/CodeGen/AMDGPU/usubo.ll
+++ b/llvm/test/CodeGen/AMDGPU/usubo.ll
@@ -866,112 +866,109 @@ define amdgpu_kernel void @v_usubo_v2i32(ptr addrspace(1) %out, ptr addrspace(1)
define amdgpu_kernel void @s_usubo_clamp_bit(ptr addrspace(1) %out, ptr addrspace(1) %carryout, i32 %a, i32 %b) #0 {
; SI-LABEL: s_usubo_clamp_bit:
; SI: ; %bb.0: ; %entry
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b64 s[4:5], 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v0, s1
-; SI-NEXT: s_cmp_eq_u32 s0, s1
-; SI-NEXT: v_sub_i32_e32 v0, vcc, s0, v0
-; SI-NEXT: s_mov_b64 s[0:1], 0
+; SI-NEXT: v_mov_b32_e32 v0, s7
+; SI-NEXT: s_cmp_eq_u32 s6, s7
+; SI-NEXT: v_sub_i32_e32 v0, vcc, s6, v0
; SI-NEXT: s_cbranch_scc1 .LBB8_2
; SI-NEXT: ; %bb.1: ; %if
-; SI-NEXT: s_xor_b64 s[0:1], vcc, -1
+; SI-NEXT: s_xor_b64 s[4:5], vcc, -1
; SI-NEXT: .LBB8_2: ; %exit
-; SI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x9
; SI-NEXT: s_mov_b32 s11, 0xf000
; SI-NEXT: s_mov_b32 s10, -1
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b32 s8, s4
-; SI-NEXT: s_mov_b32 s9, s5
-; SI-NEXT: s_mov_b32 s4, s6
-; SI-NEXT: s_mov_b32 s5, s7
-; SI-NEXT: s_mov_b32 s6, s10
-; SI-NEXT: s_mov_b32 s7, s11
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_mov_b32 s0, s2
+; SI-NEXT: s_mov_b32 s1, s3
+; SI-NEXT: s_mov_b32 s2, s10
+; SI-NEXT: s_mov_b32 s3, s11
; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[0:1]
-; SI-NEXT: buffer_store_byte v0, off, s[4:7], 0
+; SI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; SI-NEXT: buffer_store_byte v0, off, s[0:3], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: s_usubo_clamp_bit:
; VI: ; %bb.0: ; %entry
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; VI-NEXT: s_mov_b64 s[4:5], 0
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v0, s1
-; VI-NEXT: s_cmp_eq_u32 s0, s1
-; VI-NEXT: v_sub_u32_e32 v0, vcc, s0, v0
-; VI-NEXT: s_mov_b64 s[0:1], 0
+; VI-NEXT: v_mov_b32_e32 v0, s7
+; VI-NEXT: s_cmp_eq_u32 s6, s7
+; VI-NEXT: v_sub_u32_e32 v0, vcc, s6, v0
; VI-NEXT: s_cbranch_scc1 .LBB8_2
; VI-NEXT: ; %bb.1: ; %if
-; VI-NEXT: s_xor_b64 s[0:1], vcc, -1
+; VI-NEXT: s_xor_b64 s[4:5], vcc, -1
; VI-NEXT: .LBB8_2: ; %exit
-; VI-NEXT: s_load_dwordx4 s[4:7], s[4:5], 0x24
-; VI-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[0:1]
-; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: v_mov_b32_e32 v1, s4
-; VI-NEXT: v_mov_b32_e32 v2, s5
-; VI-NEXT: v_mov_b32_e32 v3, s6
-; VI-NEXT: v_mov_b32_e32 v4, s7
+; VI-NEXT: v_mov_b32_e32 v1, s0
+; VI-NEXT: v_mov_b32_e32 v2, s1
+; VI-NEXT: v_mov_b32_e32 v3, s2
+; VI-NEXT: v_mov_b32_e32 v4, s3
; VI-NEXT: flat_store_dword v[1:2], v0
-; VI-NEXT: flat_store_byte v[3:4], v5
+; VI-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; VI-NEXT: flat_store_byte v[3:4], v0
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: s_usubo_clamp_bit:
; GFX9: ; %bb.0: ; %entry
-; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX9-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX9-NEXT: s_mov_b64 s[4:5], 0
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-NEXT: s_cmp_eq_u32 s0, s1
-; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s0, v0
-; GFX9-NEXT: s_mov_b64 s[0:1], 0
+; GFX9-NEXT: v_mov_b32_e32 v0, s7
+; GFX9-NEXT: s_cmp_eq_u32 s6, s7
+; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, s6, v0
; GFX9-NEXT: s_cbranch_scc1 .LBB8_2
; GFX9-NEXT: ; %bb.1: ; %if
-; GFX9-NEXT: s_xor_b64 s[0:1], vcc, -1
+; GFX9-NEXT: s_xor_b64 s[4:5], vcc, -1
; GFX9-NEXT: .LBB8_2: ; %exit
-; GFX9-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX9-NEXT: v_mov_b32_e32 v1, 0
-; GFX9-NEXT: v_cndmask_b32_e64 v2, 0, 1, s[0:1]
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: global_store_dword v1, v0, s[8:9]
-; GFX9-NEXT: global_store_byte v1, v2, s[10:11]
+; GFX9-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX9-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
+; GFX9-NEXT: global_store_byte v1, v0, s[2:3]
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: s_usubo_clamp_bit:
; GFX10: ; %bb.0: ; %entry
-; GFX10-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x34
-; GFX10-NEXT: s_mov_b32 s0, 0
+; GFX10-NEXT: s_clause 0x1
+; GFX10-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX10-NEXT: s_mov_b32 s4, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_sub_co_u32 v0, s1, s2, s3
-; GFX10-NEXT: s_cmp_eq_u32 s2, s3
+; GFX10-NEXT: v_sub_co_u32 v0, s5, s6, s7
+; GFX10-NEXT: s_cmp_eq_u32 s6, s7
; GFX10-NEXT: s_cbranch_scc1 .LBB8_2
; GFX10-NEXT: ; %bb.1: ; %if
-; GFX10-NEXT: s_xor_b32 s0, s1, -1
+; GFX10-NEXT: s_xor_b32 s4, s5, -1
; GFX10-NEXT: .LBB8_2: ; %exit
-; GFX10-NEXT: s_load_dwordx4 s[8:11], s[4:5], 0x24
; GFX10-NEXT: v_mov_b32_e32 v1, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: global_store_dword v1, v0, s[8:9]
-; GFX10-NEXT: global_store_byte v1, v2, s[10:11]
+; GFX10-NEXT: v_cndmask_b32_e64 v2, 0, 1, s4
+; GFX10-NEXT: global_store_dword v1, v0, s[0:1]
+; GFX10-NEXT: global_store_byte v1, v2, s[2:3]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: s_usubo_clamp_bit:
; GFX11: ; %bb.0: ; %entry
-; GFX11-NEXT: s_load_b64 s[2:3], s[4:5], 0x34
-; GFX11-NEXT: s_mov_b32 s0, 0
+; GFX11-NEXT: s_clause 0x1
+; GFX11-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX11-NEXT: s_mov_b32 s4, 0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_sub_co_u32 v0, s1, s2, s3
-; GFX11-NEXT: s_cmp_eq_u32 s2, s3
+; GFX11-NEXT: v_sub_co_u32 v0, s5, s6, s7
+; GFX11-NEXT: s_cmp_eq_u32 s6, s7
; GFX11-NEXT: s_cbranch_scc1 .LBB8_2
; GFX11-NEXT: ; %bb.1: ; %if
-; GFX11-NEXT: s_xor_b32 s0, s1, -1
+; GFX11-NEXT: s_xor_b32 s4, s5, -1
; GFX11-NEXT: .LBB8_2: ; %exit
-; GFX11-NEXT: s_load_b128 s[4:7], s[4:5], 0x24
; GFX11-NEXT: v_mov_b32_e32 v1, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1, s0
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-NEXT: v_cndmask_b32_e64 v2, 0, 1, s4
; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: global_store_b32 v1, v0, s[4:5]
-; GFX11-NEXT: global_store_b8 v1, v2, s[6:7]
+; GFX11-NEXT: global_store_b32 v1, v0, s[0:1]
+; GFX11-NEXT: global_store_b8 v1, v2, s[2:3]
; GFX11-NEXT: s_endpgm
entry:
%usub = call { i32, i1 } @llvm.usub.with.overflow.i32(i32 %a, i32 %b)
diff --git a/llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll b/llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll
index 4d6ff1083492a..ff0d794fab658 100644
--- a/llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_ashr_pk.ll
@@ -7,73 +7,73 @@
define amdgpu_kernel void @v_ashr_pk_i8_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) #0 {
; GFX950-LABEL: v_ashr_pk_i8_i32:
; GFX950: ; %bb.0:
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX950-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX950-NEXT: v_mov_b32_e32 v0, 0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_and_b32 s2, s2, 31
-; GFX950-NEXT: v_mov_b32_e32 v1, s1
-; GFX950-NEXT: v_mov_b32_e32 v2, s2
-; GFX950-NEXT: v_ashr_pk_i8_i32 v1, s0, v1, v2
-; GFX950-NEXT: global_store_short v0, v1, s[6:7]
+; GFX950-NEXT: s_and_b32 s4, s6, 31
+; GFX950-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-NEXT: v_mov_b32_e32 v2, s4
+; GFX950-NEXT: v_ashr_pk_i8_i32 v1, s2, v1, v2
+; GFX950-NEXT: global_store_short v0, v1, s[0:1]
; GFX950-NEXT: s_endpgm
;
; GFX1250-TRUE16-LABEL: v_ashr_pk_i8_i32:
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_clause 0x1
-; GFX1250-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX1250-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: s_and_b32 s2, s2, 31
+; GFX1250-TRUE16-NEXT: s_and_b32 s4, s6, 31
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-TRUE16-NEXT: v_ashr_pk_i8_i32 v0.l, s0, s1, v0
-; GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, s4
+; GFX1250-TRUE16-NEXT: v_ashr_pk_i8_i32 v0.l, s2, s3, v0
+; GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: v_ashr_pk_i8_i32:
; GFX1250-FAKE16: ; %bb.0:
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_clause 0x1
-; GFX1250-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX1250-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX1250-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: s_and_b32 s2, s2, 31
+; GFX1250-FAKE16-NEXT: s_and_b32 s4, s6, 31
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-FAKE16-NEXT: v_ashr_pk_i8_i32 v0, s0, s1, v0
-; GFX1250-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s4
+; GFX1250-FAKE16-NEXT: v_ashr_pk_i8_i32 v0, s2, s3, v0
+; GFX1250-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
;
; GFX1300-TRUE16-LABEL: v_ashr_pk_i8_i32:
; GFX1300-TRUE16: ; %bb.0:
; GFX1300-TRUE16-NEXT: s_clause 0x1
-; GFX1300-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX1300-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
+; GFX1300-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1300-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1300-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1300-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1300-TRUE16-NEXT: s_and_b32 s2, s2, 31
+; GFX1300-TRUE16-NEXT: s_and_b32 s4, s6, 31
; GFX1300-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1300-TRUE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1300-TRUE16-NEXT: v_ashr_pk_i8_i32 v0.l, s0, s1, v0
-; GFX1300-TRUE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GFX1300-TRUE16-NEXT: v_mov_b32_e32 v0, s4
+; GFX1300-TRUE16-NEXT: v_ashr_pk_i8_i32 v0.l, s2, s3, v0
+; GFX1300-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1300-TRUE16-NEXT: s_endpgm
;
; GFX1300-FAKE16-LABEL: v_ashr_pk_i8_i32:
; GFX1300-FAKE16: ; %bb.0:
; GFX1300-FAKE16-NEXT: s_clause 0x1
-; GFX1300-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX1300-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
+; GFX1300-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1300-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1300-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1300-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1300-FAKE16-NEXT: s_and_b32 s2, s2, 31
+; GFX1300-FAKE16-NEXT: s_and_b32 s4, s6, 31
; GFX1300-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1300-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1300-FAKE16-NEXT: v_ashr_pk_i8_i32 v0, s0, s1, v0
-; GFX1300-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GFX1300-FAKE16-NEXT: v_mov_b32_e32 v0, s4
+; GFX1300-FAKE16-NEXT: v_ashr_pk_i8_i32 v0, s2, s3, v0
+; GFX1300-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1300-FAKE16-NEXT: s_endpgm
%insert.0 = insertelement <2 x i32> poison, i32 %src0, i64 0
%build_vector = insertelement <2 x i32> %insert.0, i32 %src1, i64 1
@@ -92,73 +92,73 @@ define amdgpu_kernel void @v_ashr_pk_i8_i32(ptr addrspace(1) %out, i32 %src0, i3
define amdgpu_kernel void @v_ashr_pk_u8_i32(ptr addrspace(1) %out, i32 %src0, i32 %src1, i32 %src2) #0 {
; GFX950-LABEL: v_ashr_pk_u8_i32:
; GFX950: ; %bb.0:
-; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX950-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX950-NEXT: s_load_dword s6, s[4:5], 0x34
+; GFX950-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX950-NEXT: v_mov_b32_e32 v0, 0
; GFX950-NEXT: s_waitcnt lgkmcnt(0)
-; GFX950-NEXT: s_and_b32 s2, s2, 31
-; GFX950-NEXT: v_mov_b32_e32 v1, s1
-; GFX950-NEXT: v_mov_b32_e32 v2, s2
-; GFX950-NEXT: v_ashr_pk_u8_i32 v1, s0, v1, v2
-; GFX950-NEXT: global_store_short v0, v1, s[6:7]
+; GFX950-NEXT: s_and_b32 s4, s6, 31
+; GFX950-NEXT: v_mov_b32_e32 v1, s3
+; GFX950-NEXT: v_mov_b32_e32 v2, s4
+; GFX950-NEXT: v_ashr_pk_u8_i32 v1, s2, v1, v2
+; GFX950-NEXT: global_store_short v0, v1, s[0:1]
; GFX950-NEXT: s_endpgm
;
; GFX1250-TRUE16-LABEL: v_ashr_pk_u8_i32:
; GFX1250-TRUE16: ; %bb.0:
; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-TRUE16-NEXT: s_clause 0x1
-; GFX1250-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX1250-TRUE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: s_and_b32 s2, s2, 31
+; GFX1250-TRUE16-NEXT: s_and_b32 s4, s6, 31
; GFX1250-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-TRUE16-NEXT: v_ashr_pk_u8_i32 v0.l, s0, s1, v0
-; GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[6:7]
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, s4
+; GFX1250-TRUE16-NEXT: v_ashr_pk_u8_i32 v0.l, s2, s3, v0
+; GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1250-TRUE16-NEXT: s_endpgm
;
; GFX1250-FAKE16-LABEL: v_ashr_pk_u8_i32:
; GFX1250-FAKE16: ; %bb.0:
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_clause 0x1
-; GFX1250-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX1250-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x24 nv
+; GFX1250-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: s_and_b32 s2, s2, 31
+; GFX1250-FAKE16-NEXT: s_and_b32 s4, s6, 31
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-FAKE16-NEXT: v_ashr_pk_u8_i32 v0, s0, s1, v0
-; GFX1250-FAKE16-NEXT: global_store_b16 v1, v0, s[6:7]
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s4
+; GFX1250-FAKE16-NEXT: v_ashr_pk_u8_i32 v0, s2, s3, v0
+; GFX1250-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
;
; GFX1300-TRUE16-LABEL: v_ashr_pk_u8_i32:
; GFX1300-TRUE16: ; %bb.0:
; GFX1300-TRUE16-NEXT: s_clause 0x1
-; GFX1300-TRUE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX1300-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
+; GFX1300-TRUE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1300-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1300-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1300-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1300-TRUE16-NEXT: s_and_b32 s2, s2, 31
+; GFX1300-TRUE16-NEXT: s_and_b32 s4, s6, 31
; GFX1300-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1300-TRUE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1300-TRUE16-NEXT: v_ashr_pk_u8_i32 v0.l, s0, s1, v0
-; GFX1300-TRUE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GFX1300-TRUE16-NEXT: v_mov_b32_e32 v0, s4
+; GFX1300-TRUE16-NEXT: v_ashr_pk_u8_i32 v0.l, s2, s3, v0
+; GFX1300-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1300-TRUE16-NEXT: s_endpgm
;
; GFX1300-FAKE16-LABEL: v_ashr_pk_u8_i32:
; GFX1300-FAKE16: ; %bb.0:
; GFX1300-FAKE16-NEXT: s_clause 0x1
-; GFX1300-FAKE16-NEXT: s_load_b96 s[0:2], s[4:5], 0x2c nv
-; GFX1300-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x24 nv
+; GFX1300-FAKE16-NEXT: s_load_b32 s6, s[4:5], 0x34 nv
+; GFX1300-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1300-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1300-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1300-FAKE16-NEXT: s_and_b32 s2, s2, 31
+; GFX1300-FAKE16-NEXT: s_and_b32 s4, s6, 31
; GFX1300-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1300-FAKE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1300-FAKE16-NEXT: v_ashr_pk_u8_i32 v0, s0, s1, v0
-; GFX1300-FAKE16-NEXT: global_store_b16 v1, v0, s[4:5]
+; GFX1300-FAKE16-NEXT: v_mov_b32_e32 v0, s4
+; GFX1300-FAKE16-NEXT: v_ashr_pk_u8_i32 v0, s2, s3, v0
+; GFX1300-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1300-FAKE16-NEXT: s_endpgm
%insert.0 = insertelement <2 x i32> poison, i32 %src0, i64 0
%build_vector = insertelement <2 x i32> %insert.0, i32 %src1, i64 1
diff --git a/llvm/test/CodeGen/AMDGPU/v_cndmask.ll b/llvm/test/CodeGen/AMDGPU/v_cndmask.ll
index 23172eb2d8158..5e958573a0bf1 100644
--- a/llvm/test/CodeGen/AMDGPU/v_cndmask.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_cndmask.ll
@@ -16,22 +16,23 @@ declare double @llvm.fabs.f64(double)
define amdgpu_kernel void @v_cnd_nan_nosgpr(ptr addrspace(1) %out, i32 %c, ptr addrspace(1) %fptr) #0 {
; SI-LABEL: v_cnd_nan_nosgpr:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s11, 0xf000
; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, s3
+; SI-NEXT: s_mov_b32 s7, s11
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_cmp_eq_u32 s8, 0
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_cmp_eq_u32 s2, 0
; SI-NEXT: s_cselect_b64 vcc, -1, 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v0, -1, v0, vcc
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v_cnd_nan_nosgpr:
@@ -42,16 +43,15 @@ define amdgpu_kernel void @v_cnd_nan_nosgpr(ptr addrspace(1) %out, i32 %c, ptr a
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: flat_load_dword v2, v[0:1]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_cmp_eq_u32 s2, 0
; VI-NEXT: s_cselect_b64 vcc, -1, 0
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -62,10 +62,8 @@ define amdgpu_kernel void @v_cnd_nan_nosgpr(ptr addrspace(1) %out, i32 %c, ptr a
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_load_dword v0, v0, s[0:1]
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_cmp_eq_u32 s2, 0
; GFX10-NEXT: s_cselect_b64 vcc, -1, 0
@@ -83,9 +81,7 @@ define amdgpu_kernel void @v_cnd_nan_nosgpr(ptr addrspace(1) %out, i32 %c, ptr a
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v0, v0, s[0:1]
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_cmp_eq_u32 s2, 0
; GFX11-NEXT: s_cselect_b64 vcc, -1, 0
@@ -146,11 +142,11 @@ define amdgpu_kernel void @v_cnd_nan(ptr addrspace(1) %out, i32 %c, float %f) #0
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_cmp_eq_u32 s2, 0
-; VI-NEXT: v_mov_b32_e32 v0, s3
+; VI-NEXT: v_mov_b32_e32 v2, s3
; VI-NEXT: s_cselect_b64 vcc, -1, 0
-; VI-NEXT: v_cndmask_b32_e32 v2, -1, v0, vcc
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: v_cndmask_b32_e32 v2, -1, v2, vcc
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -280,23 +276,22 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k1_sgprZ_f32(ptr addrspace(1) %o
define amdgpu_kernel void @fcmp_sgprX_k0_select_k1_sgprX_f32(ptr addrspace(1) %out, float %x) #0 {
; SI-LABEL: fcmp_sgprX_k0_select_k1_sgprX_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: v_cmp_nlg_f32_e64 vcc, s4, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_cmp_nlg_f32_e64 vcc, s2, 0
; SI-NEXT: v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_endpgm
;
; VI-LABEL: fcmp_sgprX_k0_select_k1_sgprX_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -310,27 +305,23 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k1_sgprX_f32(ptr addrspace(1) %o
;
; GFX10-LABEL: fcmp_sgprX_k0_select_k1_sgprX_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cmp_nlg_f32_e64 s[2:3], s6, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 1.0, s6, s[2:3]
+; GFX10-NEXT: v_cmp_nlg_f32_e64 s[4:5], s2, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v1, 1.0, s2, s[4:5]
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fcmp_sgprX_k0_select_k1_sgprX_f32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_nlg_f32_e64 s[2:3], s6, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 1.0, s6, s[2:3]
+; GFX11-NEXT: v_cmp_nlg_f32_e64 s[4:5], s2, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v1, 1.0, s2, s[4:5]
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_endpgm
;
@@ -438,23 +429,22 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_sgprZ_f32(ptr addrspace(1) %o
define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_sgprX_f32(ptr addrspace(1) %out, float %x) #0 {
; SI-LABEL: fcmp_sgprX_k0_select_k0_sgprX_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s4, s[4:5], 0xb
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v2, s4
-; SI-NEXT: v_cmp_nlg_f32_e64 vcc, s4, 0
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_mov_b32_e32 v2, s2
+; SI-NEXT: v_cmp_nlg_f32_e64 vcc, s2, 0
; SI-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_endpgm
;
; VI-LABEL: fcmp_sgprX_k0_select_k0_sgprX_f32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s1
@@ -468,27 +458,23 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_sgprX_f32(ptr addrspace(1) %o
;
; GFX10-LABEL: fcmp_sgprX_k0_select_k0_sgprX_f32:
; GFX10: ; %bb.0:
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_cmp_nlg_f32_e64 s[2:3], s6, 0
-; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, s6, s[2:3]
+; GFX10-NEXT: v_cmp_nlg_f32_e64 s[4:5], s2, 0
+; GFX10-NEXT: v_cndmask_b32_e64 v1, 0, s2, s[4:5]
; GFX10-NEXT: global_store_dword v0, v1, s[0:1]
; GFX10-NEXT: s_endpgm
;
; GFX11-LABEL: fcmp_sgprX_k0_select_k0_sgprX_f32:
; GFX11: ; %bb.0:
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_cmp_nlg_f32_e64 s[2:3], s6, 0
-; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, s6, s[2:3]
+; GFX11-NEXT: v_cmp_nlg_f32_e64 s[4:5], s2, 0
+; GFX11-NEXT: v_cndmask_b32_e64 v1, 0, s2, s[4:5]
; GFX11-NEXT: global_store_b32 v0, v1, s[0:1]
; GFX11-NEXT: s_endpgm
;
@@ -516,20 +502,19 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_sgprX_f32(ptr addrspace(1) %o
define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_vgprZ_f32(ptr addrspace(1) %out, float %x, ptr addrspace(1) %z.ptr) #0 {
; SI-LABEL: fcmp_sgprX_k0_select_k0_vgprZ_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: v_cmp_nlg_f32_e64 vcc, s6, 0
+; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_cmp_nlg_f32_e64 vcc, s2, 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v2, 0, v2, vcc
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_endpgm
;
; VI-LABEL: fcmp_sgprX_k0_select_k0_vgprZ_f32:
@@ -541,8 +526,7 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_vgprZ_f32(ptr addrspace(1) %o
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
@@ -559,10 +543,8 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_vgprZ_f32(ptr addrspace(1) %o
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cmp_nlg_f32_e64 vcc, s2, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -578,9 +560,7 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_vgprZ_f32(ptr addrspace(1) %o
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_nlg_f32_e64 vcc, s2, 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -618,20 +598,19 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k0_vgprZ_f32(ptr addrspace(1) %o
define amdgpu_kernel void @fcmp_sgprX_k0_select_k1_vgprZ_f32(ptr addrspace(1) %out, float %x, ptr addrspace(1) %z.ptr) #0 {
; SI-LABEL: fcmp_sgprX_k0_select_k1_vgprZ_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_load_dword v2, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: v_cmp_nlg_f32_e64 vcc, s6, 0
+; SI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b64 s[4:5], s[0:1]
+; SI-NEXT: v_cmp_nlg_f32_e64 vcc, s2, 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_cndmask_b32_e32 v2, 1.0, v2, vcc
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
+; SI-NEXT: buffer_store_dword v2, v[0:1], s[4:7], 0 addr64
; SI-NEXT: s_endpgm
;
; VI-LABEL: fcmp_sgprX_k0_select_k1_vgprZ_f32:
@@ -643,8 +622,7 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k1_vgprZ_f32(ptr addrspace(1) %o
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dword v3, v[0:1]
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v2
@@ -661,10 +639,8 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k1_vgprZ_f32(ptr addrspace(1) %o
; GFX10-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_load_dword v1, v0, s[0:1]
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_cmp_nlg_f32_e64 vcc, s2, 0
; GFX10-NEXT: s_waitcnt vmcnt(0)
@@ -680,9 +656,7 @@ define amdgpu_kernel void @fcmp_sgprX_k0_select_k1_vgprZ_f32(ptr addrspace(1) %o
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v1, v0, s[0:1]
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_cmp_nlg_f32_e64 vcc, s2, 0
; GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -2162,25 +2136,25 @@ define amdgpu_kernel void @fcmp_k0_vgprX_select_k1_vgprZ_f32_cond_use_x2(ptr add
define amdgpu_kernel void @v_cndmask_abs_neg_f16(ptr addrspace(1) %out, i32 %c, ptr addrspace(1) %fptr) #0 {
; SI-LABEL: v_cndmask_abs_neg_f16:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_mov_b32 s3, s7
+; SI-NEXT: s_mov_b32 s7, s11
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_load_ushort v0, v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_cmp_lg_u32 s8, 0
+; SI-NEXT: buffer_load_ushort v0, v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_cmp_lg_u32 s2, 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v1, 0x7fff, v0
; SI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; SI-NEXT: s_cselect_b64 vcc, -1, 0
; SI-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_store_short v0, off, s[4:7], 0
+; SI-NEXT: buffer_store_short v0, off, s[8:11], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v_cndmask_abs_neg_f16:
@@ -2191,18 +2165,17 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f16(ptr addrspace(1) %out, i32 %c,
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_ushort v0, v[0:1]
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: flat_load_ushort v2, v[0:1]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s2, 0
; VI-NEXT: s_cselect_b64 vcc, -1, 0
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_and_b32_e32 v1, 0x7fff, v0
-; VI-NEXT: v_xor_b32_e32 v0, 0x8000, v0
-; VI-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_and_b32_e32 v3, 0x7fff, v2
+; VI-NEXT: v_xor_b32_e32 v2, 0x8000, v2
+; VI-NEXT: v_cndmask_b32_e32 v2, v2, v3, vcc
; VI-NEXT: flat_store_short v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -2213,10 +2186,8 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f16(ptr addrspace(1) %out, i32 %c,
; GFX10-NEXT: v_mov_b32_e32 v2, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_load_ushort v0, v0, s[0:1]
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_cmp_lg_u32 s2, 0
; GFX10-NEXT: s_cselect_b64 vcc, -1, 0
@@ -2236,9 +2207,7 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f16(ptr addrspace(1) %out, i32 %c,
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_d16_b16 v0, v0, s[0:1]
-; GFX11-TRUE16-NEXT: s_clause 0x1
-; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-TRUE16-NEXT: s_cselect_b64 s[2:3], -1, 0
@@ -2259,9 +2228,7 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f16(ptr addrspace(1) %out, i32 %c,
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 1, v0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_u16 v0, v0, s[0:1]
-; GFX11-FAKE16-NEXT: s_clause 0x1
-; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-FAKE16-NEXT: s_cselect_b64 vcc, -1, 0
@@ -2328,22 +2295,23 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f16(ptr addrspace(1) %out, i32 %c,
define amdgpu_kernel void @v_cndmask_abs_neg_f32(ptr addrspace(1) %out, i32 %c, ptr addrspace(1) %fptr) #0 {
; SI-LABEL: v_cndmask_abs_neg_f32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
-; SI-NEXT: s_mov_b32 s3, 0xf000
+; SI-NEXT: s_mov_b32 s11, 0xf000
; SI-NEXT: s_mov_b32 s6, 0
-; SI-NEXT: s_mov_b32 s7, s3
+; SI-NEXT: s_mov_b32 s7, s11
; SI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: buffer_load_dword v0, v[0:1], s[4:7], 0 addr64
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: s_cmp_lg_u32 s8, 0
-; SI-NEXT: s_cselect_b64 s[4:5], -1, 0
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_cmp_lg_u32 s2, 0
+; SI-NEXT: s_cselect_b64 s[0:1], -1, 0
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: v_cndmask_b32_e64 v0, -v0, |v0|, s[4:5]
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: v_cndmask_b32_e64 v0, -v0, |v0|, s[0:1]
+; SI-NEXT: buffer_store_dword v0, off, s[8:11], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v_cndmask_abs_neg_f32:
@@ -2354,16 +2322,15 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f32(ptr addrspace(1) %out, i32 %c,
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
-; VI-NEXT: flat_load_dword v0, v[0:1]
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: flat_load_dword v2, v[0:1]
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s2, 0
-; VI-NEXT: s_cselect_b64 s[2:3], -1, 0
-; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_cndmask_b32_e64 v2, -v0, |v0|, s[2:3]
; VI-NEXT: v_mov_b32_e32 v0, s0
; VI-NEXT: v_mov_b32_e32 v1, s1
+; VI-NEXT: s_cselect_b64 s[0:1], -1, 0
+; VI-NEXT: s_waitcnt vmcnt(0)
+; VI-NEXT: v_cndmask_b32_e64 v2, -v2, |v2|, s[0:1]
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
;
@@ -2374,10 +2341,8 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f32(ptr addrspace(1) %out, i32 %c,
; GFX10-NEXT: v_mov_b32_e32 v1, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_load_dword v0, v0, s[0:1]
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_cmp_lg_u32 s2, 0
; GFX10-NEXT: s_cselect_b64 s[2:3], -1, 0
@@ -2395,9 +2360,7 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f32(ptr addrspace(1) %out, i32 %c,
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_load_b32 v0, v0, s[0:1]
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b64 s[2:3], -1, 0
@@ -2437,25 +2400,25 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f32(ptr addrspace(1) %out, i32 %c,
define amdgpu_kernel void @v_cndmask_abs_neg_f64(ptr addrspace(1) %out, i32 %c, ptr addrspace(1) %fptr) #0 {
; SI-LABEL: v_cndmask_abs_neg_f64:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s8, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0xd
-; SI-NEXT: s_mov_b32 s7, 0xf000
-; SI-NEXT: s_mov_b32 s2, 0
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0xd
+; SI-NEXT: s_mov_b32 s11, 0xf000
+; SI-NEXT: s_mov_b32 s6, 0
; SI-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: s_mov_b32 s3, s7
+; SI-NEXT: s_mov_b32 s7, s11
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[0:3], 0 addr64
-; SI-NEXT: s_load_dwordx2 s[4:5], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s6, -1
-; SI-NEXT: s_cmp_lg_u32 s8, 0
+; SI-NEXT: buffer_load_dwordx2 v[0:1], v[0:1], s[4:7], 0 addr64
+; SI-NEXT: s_mov_b32 s10, -1
+; SI-NEXT: s_mov_b32 s8, s0
+; SI-NEXT: s_mov_b32 s9, s1
+; SI-NEXT: s_cmp_lg_u32 s2, 0
; SI-NEXT: s_waitcnt vmcnt(0)
; SI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v1
; SI-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
; SI-NEXT: s_cselect_b64 vcc, -1, 0
; SI-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
+; SI-NEXT: buffer_store_dwordx2 v[0:1], off, s[8:11], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: v_cndmask_abs_neg_f64:
@@ -2467,17 +2430,16 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f64(ptr addrspace(1) %out, i32 %c,
; VI-NEXT: v_add_u32_e32 v0, vcc, s0, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_load_dwordx2 v[0:1], v[0:1]
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: s_cmp_lg_u32 s2, 0
; VI-NEXT: s_cselect_b64 vcc, -1, 0
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: s_waitcnt vmcnt(0)
-; VI-NEXT: v_and_b32_e32 v2, 0x7fffffff, v1
+; VI-NEXT: v_and_b32_e32 v4, 0x7fffffff, v1
; VI-NEXT: v_xor_b32_e32 v1, 0x80000000, v1
-; VI-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: v_cndmask_b32_e32 v1, v1, v4, vcc
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
;
@@ -2488,10 +2450,8 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f64(ptr addrspace(1) %out, i32 %c,
; GFX10-NEXT: v_mov_b32_e32 v3, 0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: global_load_dwordx2 v[0:1], v0, s[0:1]
-; GFX10-NEXT: s_clause 0x1
-; GFX10-NEXT: s_load_dword s2, s[4:5], 0x2c
; GFX10-NEXT: s_waitcnt_depctr depctr_vm_vsrc(0)
-; GFX10-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_cmp_lg_u32 s2, 0
; GFX10-NEXT: s_cselect_b64 vcc, -1, 0
@@ -2511,9 +2471,7 @@ define amdgpu_kernel void @v_cndmask_abs_neg_f64(ptr addrspace(1) %out, i32 %c,
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: global_load_b64 v[0:1], v0, s[0:1]
-; GFX11-NEXT: s_clause 0x1
-; GFX11-NEXT: s_load_b32 s2, s[4:5], 0x2c
-; GFX11-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_cmp_lg_u32 s2, 0
; GFX11-NEXT: s_cselect_b64 vcc, -1, 0
diff --git a/llvm/test/CodeGen/AMDGPU/vectorize-buffer-fat-pointer.ll b/llvm/test/CodeGen/AMDGPU/vectorize-buffer-fat-pointer.ll
index f34617e6efc55..914440e074bf0 100644
--- a/llvm/test/CodeGen/AMDGPU/vectorize-buffer-fat-pointer.ll
+++ b/llvm/test/CodeGen/AMDGPU/vectorize-buffer-fat-pointer.ll
@@ -7,7 +7,7 @@ entry:
%a2 = getelementptr i32, ptr addrspace(7) %out, i32 2
%a3 = getelementptr i32, ptr addrspace(7) %out, i32 3
-; OPT: store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, ptr addrspace(7) %out, align 16
+; OPT: store <4 x b32> <b32 0, b32 1, b32 2, b32 3>, ptr addrspace(7) %out, align 16
store i32 0, ptr addrspace(7) %out, align 16
store i32 1, ptr addrspace(7) %a1, align 4
store i32 2, ptr addrspace(7) %a2, align 8
@@ -22,7 +22,7 @@ entry:
%a2 = getelementptr i32, ptr addrspace(9) %out, i32 2
%a3 = getelementptr i32, ptr addrspace(9) %out, i32 3
-; OPT: store <4 x i32> <i32 0, i32 1, i32 2, i32 3>, ptr addrspace(9) %out, align 16
+; OPT: store <4 x b32> <b32 0, b32 1, b32 2, b32 3>, ptr addrspace(9) %out, align 16
store i32 0, ptr addrspace(9) %out, align 16
store i32 1, ptr addrspace(9) %a1, align 4
store i32 2, ptr addrspace(9) %a2, align 8
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll
index d96228141956f..d8a8022d230af 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-flushed.ll
@@ -9,8 +9,8 @@ define amdgpu_kernel void @ds_prefetch_flushed(ptr addrspace(3) %lds, ptr addrsp
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: s_clause 0x1
-; CHECK-NEXT: s_load_b32 s1, s[4:5], 0x0 nv
-; CHECK-NEXT: s_load_b32 s0, s[4:5], 0x10 nv
+; CHECK-NEXT: s_load_b32 s3, s[4:5], 0x0 nv
+; CHECK-NEXT: s_load_b96 s[0:2], s[4:5], 0x8 nv
; CHECK-NEXT: v_and_b32_e32 v10, 0x3ff, v0
; CHECK-NEXT: v_mov_b32_e32 v4, 0
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
@@ -18,11 +18,11 @@ define amdgpu_kernel void @ds_prefetch_flushed(ptr addrspace(3) %lds, ptr addrsp
; CHECK-NEXT: v_dual_mov_b32 v7, v4 :: v_dual_mov_b32 v8, v4
; CHECK-NEXT: v_mov_b32_e32 v9, v4
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_lshl_add_u32 v11, v10, 6, s1
-; CHECK-NEXT: v_lshl_add_u32 v12, v10, 5, s1
-; CHECK-NEXT: v_lshl_add_u32 v13, v10, 8, s1
-; CHECK-NEXT: v_lshl_add_u32 v14, v10, 7, s1
-; CHECK-NEXT: s_mov_b32 s1, 0
+; CHECK-NEXT: v_lshl_add_u32 v11, v10, 6, s3
+; CHECK-NEXT: v_lshl_add_u32 v12, v10, 5, s3
+; CHECK-NEXT: v_lshl_add_u32 v13, v10, 8, s3
+; CHECK-NEXT: v_lshl_add_u32 v14, v10, 7, s3
+; CHECK-NEXT: s_mov_b32 s3, 0
; CHECK-NEXT: ds_load_b64 v[0:1], v11 offset:4
; CHECK-NEXT: ds_load_b64 v[2:3], v12
; CHECK-NEXT: s_wait_dscnt 0x0
@@ -31,8 +31,8 @@ define amdgpu_kernel void @ds_prefetch_flushed(ptr addrspace(3) %lds, ptr addrsp
; CHECK-NEXT: s_barrier_signal -1
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; CHECK-NEXT: v_pk_add_f32 v[8:9], v[8:9], v[2:3]
-; CHECK-NEXT: s_add_co_i32 s1, s1, 1
-; CHECK-NEXT: s_cmp_lt_i32 s1, s0
+; CHECK-NEXT: s_add_co_i32 s3, s3, 1
+; CHECK-NEXT: s_cmp_lt_i32 s3, s2
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; CHECK-NEXT: v_pk_add_f32 v[8:9], v[8:9], v[0:1]
; CHECK-NEXT: s_wait_dscnt 0x1
@@ -53,8 +53,6 @@ define amdgpu_kernel void @ds_prefetch_flushed(ptr addrspace(3) %lds, ptr addrsp
; CHECK-NEXT: v_pk_add_f32 v[8:9], v[8:9], v[16:17]
; CHECK-NEXT: s_cbranch_scc1 .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %exit
-; CHECK-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
-; CHECK-NEXT: s_wait_kmcnt 0x0
; CHECK-NEXT: global_store_b64 v10, v[8:9], s[0:1] scale_offset
; CHECK-NEXT: s_endpgm
entry:
diff --git a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
index 7f98f8cf8de06..17827ca0e030b 100644
--- a/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
+++ b/llvm/test/CodeGen/AMDGPU/waitcnt-loop-ds-prefetch-pattern.ll
@@ -9,42 +9,40 @@ define amdgpu_kernel void @ds_prefetch_pattern(ptr addrspace(3) %lds, ptr addrsp
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; CHECK-NEXT: s_clause 0x1
-; CHECK-NEXT: s_load_b32 s1, s[4:5], 0x0 nv
-; CHECK-NEXT: s_load_b32 s0, s[4:5], 0x10 nv
+; CHECK-NEXT: s_load_b32 s3, s[4:5], 0x0 nv
+; CHECK-NEXT: s_load_b96 s[0:2], s[4:5], 0x8 nv
; CHECK-NEXT: v_and_b32_e32 v12, 0x3ff, v0
-; CHECK-NEXT: v_mov_b32_e32 v4, 0
+; CHECK-NEXT: v_mov_b32_e32 v0, 0
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; CHECK-NEXT: v_dual_mov_b32 v5, v4 :: v_dual_mov_b32 v6, v4
-; CHECK-NEXT: v_mov_b32_e32 v7, v4
+; CHECK-NEXT: v_dual_mov_b32 v1, v0 :: v_dual_mov_b32 v2, v0
+; CHECK-NEXT: v_mov_b32_e32 v3, v0
; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: v_lshl_add_u32 v13, v12, 8, s1
-; CHECK-NEXT: s_mov_b32 s1, 0
+; CHECK-NEXT: v_lshl_add_u32 v13, v12, 8, s3
+; CHECK-NEXT: s_mov_b32 s3, 0
; CHECK-NEXT: ds_load_b128 v[8:11], v13
-; CHECK-NEXT: ds_load_b128 v[0:3], v13 offset:16
+; CHECK-NEXT: ds_load_b128 v[4:7], v13 offset:16
; CHECK-NEXT: s_wait_dscnt 0x0
; CHECK-NEXT: .LBB0_1: ; %loop
; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1
; CHECK-NEXT: s_barrier_signal -1
-; CHECK-NEXT: s_add_co_i32 s1, s1, 1
+; CHECK-NEXT: s_add_co_i32 s3, s3, 1
; CHECK-NEXT: s_wait_dscnt 0x1
-; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[10:11]
-; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[8:9]
-; CHECK-NEXT: v_lshl_add_u32 v14, s1, 5, v13
-; CHECK-NEXT: s_cmp_lt_i32 s1, s0
+; CHECK-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[10:11]
+; CHECK-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[8:9]
+; CHECK-NEXT: v_lshl_add_u32 v14, s3, 5, v13
+; CHECK-NEXT: s_cmp_lt_i32 s3, s2
; CHECK-NEXT: s_wait_dscnt 0x0
-; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[2:3]
-; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[0:1]
+; CHECK-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[6:7]
+; CHECK-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[4:5]
; CHECK-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; CHECK-NEXT: v_pk_add_f32 v[6:7], v[6:7], v[6:7]
-; CHECK-NEXT: v_pk_add_f32 v[4:5], v[4:5], v[4:5]
+; CHECK-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[2:3]
+; CHECK-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[0:1]
; CHECK-NEXT: s_barrier_wait -1
; CHECK-NEXT: ds_load_b128 v[8:11], v14
-; CHECK-NEXT: ds_load_b128 v[0:3], v14 offset:16
+; CHECK-NEXT: ds_load_b128 v[4:7], v14 offset:16
; CHECK-NEXT: s_cbranch_scc1 .LBB0_1
; CHECK-NEXT: ; %bb.2: ; %exit
-; CHECK-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
-; CHECK-NEXT: s_wait_kmcnt 0x0
-; CHECK-NEXT: global_store_b128 v12, v[4:7], s[0:1] scale_offset
+; CHECK-NEXT: global_store_b128 v12, v[0:3], s[0:1] scale_offset
; CHECK-NEXT: s_endpgm
entry:
%tid = call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/wave32.ll b/llvm/test/CodeGen/AMDGPU/wave32.ll
index 013b402a82488..4ac46923a9688 100644
--- a/llvm/test/CodeGen/AMDGPU/wave32.ll
+++ b/llvm/test/CodeGen/AMDGPU/wave32.ll
@@ -137,9 +137,7 @@ define amdgpu_kernel void @test_vopc_2xf16(ptr addrspace(1) %arg) #1 {
define amdgpu_kernel void @test_vopc_class(ptr addrspace(1) %out, float %x) #0 {
; GFX1032-LABEL: test_vopc_class:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_clause 0x1
-; GFX1032-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_cmp_class_f32_e64 s2, s2, 0x204
@@ -149,9 +147,7 @@ define amdgpu_kernel void @test_vopc_class(ptr addrspace(1) %out, float %x) #0 {
;
; GFX1064-LABEL: test_vopc_class:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_clause 0x1
-; GFX1064-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: v_mov_b32_e32 v0, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_cmp_class_f32_e64 s[2:3], s2, 0x204
@@ -168,9 +164,7 @@ define amdgpu_kernel void @test_vopc_class(ptr addrspace(1) %out, float %x) #0 {
define amdgpu_kernel void @test_vcmp_vcnd_f16(ptr addrspace(1) %out, half %x) #0 {
; GFX1032-LABEL: test_vcmp_vcnd_f16:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_clause 0x1
-; GFX1032-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_mov_b32_e32 v0, s2
@@ -181,9 +175,7 @@ define amdgpu_kernel void @test_vcmp_vcnd_f16(ptr addrspace(1) %out, half %x) #0
;
; GFX1064-LABEL: test_vcmp_vcnd_f16:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_clause 0x1
-; GFX1064-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_mov_b32_e32 v0, s2
@@ -1169,31 +1161,31 @@ define amdgpu_kernel void @test_div_fmas_f32(ptr addrspace(1) %out, float %a, fl
; GFX1032-LABEL: test_div_fmas_f32:
; GFX1032: ; %bb.0:
; GFX1032-NEXT: s_clause 0x1
-; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: v_mov_b32_e32 v2, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1032-NEXT: v_mov_b32_e32 v0, s1
-; GFX1032-NEXT: v_mov_b32_e32 v1, s2
-; GFX1032-NEXT: s_bitcmp1_b32 s3, 0
+; GFX1032-NEXT: v_mov_b32_e32 v1, s6
+; GFX1032-NEXT: v_mov_b32_e32 v0, s3
+; GFX1032-NEXT: s_bitcmp1_b32 s7, 0
; GFX1032-NEXT: s_cselect_b32 vcc_lo, -1, 0
-; GFX1032-NEXT: v_div_fmas_f32 v0, s0, v0, v1
-; GFX1032-NEXT: global_store_dword v2, v0, s[6:7]
+; GFX1032-NEXT: v_div_fmas_f32 v0, s2, v0, v1
+; GFX1032-NEXT: global_store_dword v2, v0, s[0:1]
; GFX1032-NEXT: s_endpgm
;
; GFX1064-LABEL: test_div_fmas_f32:
; GFX1064: ; %bb.0:
; GFX1064-NEXT: s_clause 0x1
-; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x2c
-; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0x34
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_mov_b32_e32 v0, s1
-; GFX1064-NEXT: v_mov_b32_e32 v1, s2
-; GFX1064-NEXT: s_bitcmp1_b32 s3, 0
+; GFX1064-NEXT: v_mov_b32_e32 v1, s6
+; GFX1064-NEXT: v_mov_b32_e32 v0, s3
+; GFX1064-NEXT: s_bitcmp1_b32 s7, 0
; GFX1064-NEXT: s_cselect_b64 vcc, -1, 0
-; GFX1064-NEXT: v_div_fmas_f32 v0, s0, v0, v1
-; GFX1064-NEXT: global_store_dword v2, v0, s[6:7]
+; GFX1064-NEXT: v_div_fmas_f32 v0, s2, v0, v1
+; GFX1064-NEXT: global_store_dword v2, v0, s[0:1]
; GFX1064-NEXT: s_endpgm
%result = call float @llvm.amdgcn.div.fmas.f32(float %a, float %b, float %c, i1 %d) nounwind readnone
store float %result, ptr addrspace(1) %out, align 4
@@ -1652,11 +1644,9 @@ entry:
define amdgpu_kernel void @test_set_inactive(ptr addrspace(1) %out, i32 %in) #0 {
; GFX1032-LABEL: test_set_inactive:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_clause 0x1
-; GFX1032-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1032-NEXT: s_or_saveexec_b32 s3, -1
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1032-NEXT: s_or_saveexec_b32 s3, -1
; GFX1032-NEXT: v_cndmask_b32_e64 v0, 42, s2, s3
; GFX1032-NEXT: s_mov_b32 exec_lo, s3
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
@@ -1666,13 +1656,11 @@ define amdgpu_kernel void @test_set_inactive(ptr addrspace(1) %out, i32 %in) #0
;
; GFX1064-LABEL: test_set_inactive:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_clause 0x1
-; GFX1064-NEXT: s_load_dword s6, s[4:5], 0x2c
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX1064-NEXT: s_or_saveexec_b64 s[2:3], -1
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
+; GFX1064-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1064-NEXT: v_cndmask_b32_e64 v0, 42, s6, s[2:3]
-; GFX1064-NEXT: s_mov_b64 exec, s[2:3]
+; GFX1064-NEXT: v_cndmask_b32_e64 v0, 42, s2, s[4:5]
+; GFX1064-NEXT: s_mov_b64 exec, s[4:5]
; GFX1064-NEXT: v_mov_b32_e32 v1, 0
; GFX1064-NEXT: v_mov_b32_e32 v2, v0
; GFX1064-NEXT: global_store_dword v1, v2, s[0:1]
@@ -2148,9 +2136,7 @@ define amdgpu_kernel void @test_intr_fcmp_i64(ptr addrspace(1) %out, float %src,
define amdgpu_kernel void @test_intr_icmp_i64(ptr addrspace(1) %out, i32 %src) #1 {
; GFX1032-LABEL: test_intr_icmp_i64:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_clause 0x1
-; GFX1032-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: v_mov_b32_e32 v1, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e64 s2, 0x64, s2
@@ -2160,9 +2146,7 @@ define amdgpu_kernel void @test_intr_icmp_i64(ptr addrspace(1) %out, i32 %src) #
;
; GFX1064-LABEL: test_intr_icmp_i64:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_clause 0x1
-; GFX1064-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: v_mov_b32_e32 v2, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e64 s[2:3], 0x64, s2
@@ -2204,9 +2188,7 @@ define amdgpu_kernel void @test_intr_fcmp_i32(ptr addrspace(1) %out, float %src,
define amdgpu_kernel void @test_intr_icmp_i32(ptr addrspace(1) %out, i32 %src) #1 {
; GFX1032-LABEL: test_intr_icmp_i32:
; GFX1032: ; %bb.0:
-; GFX1032-NEXT: s_clause 0x1
-; GFX1032-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1032-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1032-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1032-NEXT: v_mov_b32_e32 v0, 0
; GFX1032-NEXT: s_waitcnt lgkmcnt(0)
; GFX1032-NEXT: v_cmp_eq_u32_e64 s2, 0x64, s2
@@ -2216,9 +2198,7 @@ define amdgpu_kernel void @test_intr_icmp_i32(ptr addrspace(1) %out, i32 %src) #
;
; GFX1064-LABEL: test_intr_icmp_i32:
; GFX1064: ; %bb.0:
-; GFX1064-NEXT: s_clause 0x1
-; GFX1064-NEXT: s_load_dword s2, s[4:5], 0x2c
-; GFX1064-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX1064-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX1064-NEXT: v_mov_b32_e32 v0, 0
; GFX1064-NEXT: s_waitcnt lgkmcnt(0)
; GFX1064-NEXT: v_cmp_eq_u32_e64 s[2:3], 0x64, s2
diff --git a/llvm/test/CodeGen/AMDGPU/xor.ll b/llvm/test/CodeGen/AMDGPU/xor.ll
index 92280b9ad8acf..1c9be2546d133 100644
--- a/llvm/test/CodeGen/AMDGPU/xor.ll
+++ b/llvm/test/CodeGen/AMDGPU/xor.ll
@@ -298,23 +298,24 @@ define amdgpu_kernel void @scalar_xor_i32(ptr addrspace(1) %out, i32 %a, i32 %b)
; SI-LABEL: scalar_xor_i32:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_mov_b64 s[4:5], s[2:3]
-; SI-NEXT: s_xor_b32 s4, s4, s5
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_xor_b32 s0, s2, s3
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: scalar_xor_i32:
; VI: ; %bb.0:
; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_xor_b32 s2, s2, s3
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_xor_b32 s0, s2, s3
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%result = xor i32 %a, %b
@@ -325,25 +326,25 @@ define amdgpu_kernel void @scalar_xor_i32(ptr addrspace(1) %out, i32 %a, i32 %b)
define amdgpu_kernel void @scalar_not_i32(ptr addrspace(1) %out, i32 %a) {
; SI-LABEL: scalar_not_i32:
; SI: ; %bb.0:
-; SI-NEXT: s_load_dword s6, s[4:5], 0xb
-; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; SI-NEXT: s_mov_b32 s3, 0xf000
-; SI-NEXT: s_mov_b32 s2, -1
+; SI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; SI-NEXT: s_mov_b32 s7, 0xf000
+; SI-NEXT: s_mov_b32 s6, -1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: s_not_b32 s4, s6
-; SI-NEXT: v_mov_b32_e32 v0, s4
-; SI-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; SI-NEXT: s_mov_b32 s4, s0
+; SI-NEXT: s_not_b32 s0, s2
+; SI-NEXT: s_mov_b32 s5, s1
+; SI-NEXT: v_mov_b32_e32 v0, s0
+; SI-NEXT: buffer_store_dword v0, off, s[4:7], 0
; SI-NEXT: s_endpgm
;
; VI-LABEL: scalar_not_i32:
; VI: ; %bb.0:
-; VI-NEXT: s_load_dword s2, s[4:5], 0x2c
-; VI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; VI-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NEXT: s_not_b32 s2, s2
; VI-NEXT: v_mov_b32_e32 v0, s0
+; VI-NEXT: s_not_b32 s0, s2
; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
%result = xor i32 %a, -1
diff --git a/llvm/test/CodeGen/AMDGPU/zext-divergence-driven-isel.ll b/llvm/test/CodeGen/AMDGPU/zext-divergence-driven-isel.ll
index d9f5ba92e116d..524679d42e73e 100644
--- a/llvm/test/CodeGen/AMDGPU/zext-divergence-driven-isel.ll
+++ b/llvm/test/CodeGen/AMDGPU/zext-divergence-driven-isel.ll
@@ -5,14 +5,15 @@ define amdgpu_kernel void @zext_i16_to_i32_uniform(ptr addrspace(1) %out, i16 %a
; GCN-LABEL: zext_i16_to_i32_uniform:
; GCN: ; %bb.0:
; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_mov_b64 s[4:5], s[2:3]
-; GCN-NEXT: s_and_b32 s4, s4, 0xffff
-; GCN-NEXT: s_add_i32 s4, s5, s4
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_and_b32 s0, s2, 0xffff
+; GCN-NEXT: s_add_i32 s0, s3, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%zext = zext i16 %a to i32
%res = add i32 %b, %zext
@@ -24,18 +25,19 @@ define amdgpu_kernel void @zext_i16_to_i32_uniform(ptr addrspace(1) %out, i16 %a
define amdgpu_kernel void @zext_i16_to_i64_uniform(ptr addrspace(1) %out, i16 %a, i64 %b) {
; GCN-LABEL: zext_i16_to_i64_uniform:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s8, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[6:7], s[4:5], 0xd
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_load_dwordx2 s[8:9], s[4:5], 0xd
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: s_and_b32 s4, s8, 0xffff
-; GCN-NEXT: s_add_u32 s4, s6, s4
-; GCN-NEXT: s_addc_u32 s5, s7, 0
-; GCN-NEXT: v_mov_b32_e32 v0, s4
-; GCN-NEXT: v_mov_b32_e32 v1, s5
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_and_b32 s0, s2, 0xffff
+; GCN-NEXT: s_add_u32 s0, s8, s0
+; GCN-NEXT: s_mov_b32 s5, s1
+; GCN-NEXT: s_addc_u32 s1, s9, 0
+; GCN-NEXT: v_mov_b32_e32 v0, s0
+; GCN-NEXT: v_mov_b32_e32 v1, s1
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-NEXT: s_endpgm
%zext = zext i16 %a to i64
%res = add i64 %b, %zext
@@ -46,14 +48,15 @@ define amdgpu_kernel void @zext_i16_to_i64_uniform(ptr addrspace(1) %out, i16 %a
define amdgpu_kernel void @zext_i16_to_i32_divergent(ptr addrspace(1) %out, i16 %a, i32 %b) {
; GCN-LABEL: zext_i16_to_i32_divergent:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GCN-NEXT: buffer_store_dword v0, off, s[0:3], 0
+; GCN-NEXT: buffer_store_dword v0, off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.truncated = trunc i32 %tid to i16
@@ -67,15 +70,16 @@ define amdgpu_kernel void @zext_i16_to_i32_divergent(ptr addrspace(1) %out, i16
define amdgpu_kernel void @zext_i16_to_i64_divergent(ptr addrspace(1) %out, i16 %a, i64 %b) {
; GCN-LABEL: zext_i16_to_i64_divergent:
; GCN: ; %bb.0:
-; GCN-NEXT: s_load_dword s6, s[4:5], 0xb
-; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
-; GCN-NEXT: s_mov_b32 s3, 0xf000
-; GCN-NEXT: s_mov_b32 s2, -1
+; GCN-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x9
+; GCN-NEXT: s_mov_b32 s7, 0xf000
+; GCN-NEXT: s_mov_b32 s6, -1
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_i32_e32 v0, vcc, s6, v0
+; GCN-NEXT: v_add_i32_e32 v0, vcc, s2, v0
+; GCN-NEXT: s_mov_b32 s4, s0
+; GCN-NEXT: s_mov_b32 s5, s1
; GCN-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
+; GCN-NEXT: buffer_store_dwordx2 v[0:1], off, s[4:7], 0
; GCN-NEXT: s_endpgm
%tid = call i32 @llvm.amdgcn.workitem.id.x()
%tid.truncated = trunc i32 %tid to i16
diff --git a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
index e408458d65f1a..5f3761faf6a2c 100644
--- a/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
+++ b/llvm/test/CodeGen/MIR/AMDGPU/machine-function-info-long-branch-reg.ll
@@ -43,7 +43,7 @@
; CHECK-NEXT: occupancy: 10
; CHECK-NEXT: vgprForAGPRCopy: ''
; CHECK-NEXT: sgprForEXECCopy: '$sgpr100_sgpr101'
-; CHECK-NEXT: longBranchReservedReg: '$sgpr2_sgpr3'
+; CHECK-NEXT: longBranchReservedReg: '$sgpr6_sgpr7'
; CHECK-NEXT: hasInitWholeWave: false
; CHECK-NEXT: dynamicVGPRBlockSize: 0
; CHECK-NEXT: scratchReservedForDynamicVGPRs: 0
diff --git a/llvm/test/CodeGen/NVPTX/dag-cse.ll b/llvm/test/CodeGen/NVPTX/dag-cse.ll
index 84a38fd30963b..57520e226aa48 100644
--- a/llvm/test/CodeGen/NVPTX/dag-cse.ll
+++ b/llvm/test/CodeGen/NVPTX/dag-cse.ll
@@ -9,11 +9,9 @@
; Verify that loads with different memory types are not subject to CSE
; once they are promoted to the same type.
;
-; CHECK: ld.global.v2.b8 {%[[B1:rs[0-9]+]], %[[B2:rs[0-9]+]]}, [a];
-; CHECK: st.global.v2.b8 [b], {%[[B1]], %[[B2]]};
-;
; CHECK: ld.global.b32 %[[C:r[0-9]+]], [a];
-; CHECK: st.global.b32 [c], %[[C]];
+; CHECK: st.global.v2.b8 [b],
+; CHECK: st.global.b32 [c],
define void @test1() #0 {
%1 = load <2 x i8>, ptr addrspace(1) @a, align 8
diff --git a/llvm/test/CodeGen/NVPTX/variadics-backend.ll b/llvm/test/CodeGen/NVPTX/variadics-backend.ll
index 61ff80632c789..f1a78d11fd0f8 100644
--- a/llvm/test/CodeGen/NVPTX/variadics-backend.ll
+++ b/llvm/test/CodeGen/NVPTX/variadics-backend.ll
@@ -142,7 +142,7 @@ define dso_local i32 @variadics2(i32 noundef %first, ...) {
; CHECK-PTX-NEXT: .reg .b64 %SP;
; CHECK-PTX-NEXT: .reg .b64 %SPL;
; CHECK-PTX-NEXT: .reg .b16 %rs<4>;
-; CHECK-PTX-NEXT: .reg .b32 %r<6>;
+; CHECK-PTX-NEXT: .reg .b32 %r<7>;
; CHECK-PTX-NEXT: .reg .b64 %rd<8>;
; CHECK-PTX-EMPTY:
; CHECK-PTX-NEXT: // %bb.0: // %entry
@@ -152,8 +152,7 @@ define dso_local i32 @variadics2(i32 noundef %first, ...) {
; CHECK-PTX-NEXT: add.u64 %rd2, %SPL, 0;
; CHECK-PTX-NEXT: add.s64 %rd3, %rd1, 7;
; CHECK-PTX-NEXT: and.b64 %rd4, %rd3, -8;
-; CHECK-PTX-NEXT: ld.b32 %r2, [%rd4];
-; CHECK-PTX-NEXT: ld.s8 %r3, [%rd4+4];
+; CHECK-PTX-NEXT: ld.v2.b32 {%r2, %r3}, [%rd4];
; CHECK-PTX-NEXT: ld.b8 %rs1, [%rd4+7];
; CHECK-PTX-NEXT: st.local.b8 [%rd2+2], %rs1;
; CHECK-PTX-NEXT: ld.b8 %rs2, [%rd4+6];
@@ -162,8 +161,9 @@ define dso_local i32 @variadics2(i32 noundef %first, ...) {
; CHECK-PTX-NEXT: st.local.b8 [%rd2], %rs3;
; CHECK-PTX-NEXT: ld.b64 %rd5, [%rd4+8];
; CHECK-PTX-NEXT: add.s32 %r4, %r1, %r2;
-; CHECK-PTX-NEXT: add.s32 %r5, %r4, %r3;
-; CHECK-PTX-NEXT: cvt.u64.u32 %rd6, %r5;
+; CHECK-PTX-NEXT: cvt.s32.s8 %r5, %r3;
+; CHECK-PTX-NEXT: add.s32 %r6, %r4, %r5;
+; CHECK-PTX-NEXT: cvt.u64.u32 %rd6, %r6;
; CHECK-PTX-NEXT: add.s64 %rd7, %rd6, %rd5;
; CHECK-PTX-NEXT: st.param.b32 [func_retval0], %rd7;
; CHECK-PTX-NEXT: ret;
diff --git a/llvm/test/CodeGen/X86/loadStore_vectorizer.ll b/llvm/test/CodeGen/X86/loadStore_vectorizer.ll
index 7a75479efc2e7..df56822a98bbd 100644
--- a/llvm/test/CodeGen/X86/loadStore_vectorizer.ll
+++ b/llvm/test/CodeGen/X86/loadStore_vectorizer.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple x86_64-- -passes=load-store-vectorizer < %s -S | FileCheck %s
%struct_render_pipeline_state = type opaque
@@ -15,7 +16,7 @@ entry:
define fastcc void @test2(ptr addrspace(1) %pso) unnamed_addr {
; CHECK-LABEL: @test2
-; CHECK: load <2 x i16>
+; CHECK: load <2 x b16>
entry:
%tmp1 = load i16, ptr addrspace(1) %pso, align 2
%sunkaddr51 = getelementptr i8, ptr addrspace(1) %pso, i64 2
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
index eab802a9be813..7df06af5e89d6 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/aa-metadata.ll
@@ -7,18 +7,18 @@
define amdgpu_kernel void @vectorize_alias_scope(ptr addrspace(1) nocapture %a, ptr addrspace(1) nocapture %b, ptr addrspace(1) nocapture readonly %c) #0 {
; SCOPE-LABEL: @vectorize_alias_scope(
; SCOPE-NEXT: entry:
-; SCOPE-NEXT: [[LD_C:%.*]] = load float, ptr addrspace(1) [[C:%.*]], align 4, !alias.scope !0
-; SCOPE-NEXT: store <2 x float> zeroinitializer, ptr addrspace(1) [[A:%.*]], align 4, !noalias !0
-; SCOPE-NEXT: store float [[LD_C]], ptr addrspace(1) [[B:%.*]], align 4, !noalias !0
+; SCOPE-NEXT: [[LD_C:%.*]] = load float, ptr addrspace(1) [[C:%.*]], align 4, !alias.scope [[META0:![0-9]+]]
+; SCOPE-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(1) [[A:%.*]], align 4, !noalias [[META0]]
+; SCOPE-NEXT: store float [[LD_C]], ptr addrspace(1) [[B:%.*]], align 4, !noalias [[META0]]
; SCOPE-NEXT: ret void
;
; NOSCOPE-LABEL: @vectorize_alias_scope(
; NOSCOPE-NEXT: entry:
; NOSCOPE-NEXT: [[A_IDX_1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A:%.*]], i64 1
-; NOSCOPE-NEXT: store float 0.000000e+00, ptr addrspace(1) [[A]], align 4, !noalias !0
-; NOSCOPE-NEXT: [[LD_C:%.*]] = load float, ptr addrspace(1) [[C:%.*]], align 4, !alias.scope !0
-; NOSCOPE-NEXT: store float 0.000000e+00, ptr addrspace(1) [[A_IDX_1]], align 4, !noalias !0
-; NOSCOPE-NEXT: store float [[LD_C]], ptr addrspace(1) [[B:%.*]], align 4, !noalias !0
+; NOSCOPE-NEXT: store float 0.000000e+00, ptr addrspace(1) [[A]], align 4, !noalias [[META0:![0-9]+]]
+; NOSCOPE-NEXT: [[LD_C:%.*]] = load float, ptr addrspace(1) [[C:%.*]], align 4, !alias.scope [[META0]]
+; NOSCOPE-NEXT: store float 0.000000e+00, ptr addrspace(1) [[A_IDX_1]], align 4, !noalias [[META0]]
+; NOSCOPE-NEXT: store float [[LD_C]], ptr addrspace(1) [[B:%.*]], align 4, !noalias [[META0]]
; NOSCOPE-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
index 6ad5e002a5ddd..8ccd6e925236c 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/adjust-alloca-alignment.ll
@@ -20,9 +20,11 @@ define amdgpu_kernel void @load_unknown_offset_align1_i8(ptr addrspace(1) noalia
; UNALIGNED-LABEL: @load_unknown_offset_align1_i8(
; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i8], align 1, addrspace(5)
; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i8], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]]
-; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr addrspace(5) [[PTR0]], align 1
-; UNALIGNED-NEXT: [[VAL01:%.*]] = extractelement <2 x i8> [[TMP1]], i32 0
-; UNALIGNED-NEXT: [[VAL12:%.*]] = extractelement <2 x i8> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x b8>, ptr addrspace(5) [[PTR0]], align 1
+; UNALIGNED-NEXT: [[VAL1:%.*]] = extractelement <2 x b8> [[TMP1]], i32 0
+; UNALIGNED-NEXT: [[VAL01:%.*]] = bitcast b8 [[VAL1]] to i8
+; UNALIGNED-NEXT: [[VAL13:%.*]] = extractelement <2 x b8> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[VAL12:%.*]] = bitcast b8 [[VAL13]] to i8
; UNALIGNED-NEXT: [[ADD:%.*]] = add i8 [[VAL01]], [[VAL12]]
; UNALIGNED-NEXT: store i8 [[ADD]], ptr addrspace(1) [[OUT:%.*]], align 1
; UNALIGNED-NEXT: ret void
@@ -51,9 +53,11 @@ define amdgpu_kernel void @load_unknown_offset_align1_i16(ptr addrspace(1) noali
; UNALIGNED-LABEL: @load_unknown_offset_align1_i16(
; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i16], align 1, addrspace(5)
; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i16], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]]
-; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i16>, ptr addrspace(5) [[PTR0]], align 1
-; UNALIGNED-NEXT: [[VAL01:%.*]] = extractelement <2 x i16> [[TMP1]], i32 0
-; UNALIGNED-NEXT: [[VAL12:%.*]] = extractelement <2 x i16> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x b16>, ptr addrspace(5) [[PTR0]], align 1
+; UNALIGNED-NEXT: [[VAL1:%.*]] = extractelement <2 x b16> [[TMP1]], i32 0
+; UNALIGNED-NEXT: [[VAL01:%.*]] = bitcast b16 [[VAL1]] to i16
+; UNALIGNED-NEXT: [[VAL13:%.*]] = extractelement <2 x b16> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[VAL12:%.*]] = bitcast b16 [[VAL13]] to i16
; UNALIGNED-NEXT: [[ADD:%.*]] = add i16 [[VAL01]], [[VAL12]]
; UNALIGNED-NEXT: store i16 [[ADD]], ptr addrspace(1) [[OUT:%.*]], align 2
; UNALIGNED-NEXT: ret void
@@ -84,9 +88,11 @@ define amdgpu_kernel void @load_unknown_offset_align1_i32(ptr addrspace(1) noali
; UNALIGNED-LABEL: @load_unknown_offset_align1_i32(
; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i32], align 1, addrspace(5)
; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]]
-; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(5) [[PTR0]], align 1
-; UNALIGNED-NEXT: [[VAL01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; UNALIGNED-NEXT: [[VAL12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(5) [[PTR0]], align 1
+; UNALIGNED-NEXT: [[VAL1:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; UNALIGNED-NEXT: [[VAL01:%.*]] = bitcast b32 [[VAL1]] to i32
+; UNALIGNED-NEXT: [[VAL13:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[VAL12:%.*]] = bitcast b32 [[VAL13]] to i32
; UNALIGNED-NEXT: [[ADD:%.*]] = add i32 [[VAL01]], [[VAL12]]
; UNALIGNED-NEXT: store i32 [[ADD]], ptr addrspace(1) [[OUT:%.*]], align 4
; UNALIGNED-NEXT: ret void
@@ -116,9 +122,11 @@ define amdgpu_kernel void @load_alloca16_unknown_offset_align1_i32(ptr addrspace
; UNALIGNED-LABEL: @load_alloca16_unknown_offset_align1_i32(
; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i32], align 16, addrspace(5)
; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]]
-; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(5) [[PTR0]], align 4
-; UNALIGNED-NEXT: [[VAL01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; UNALIGNED-NEXT: [[VAL12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(5) [[PTR0]], align 4
+; UNALIGNED-NEXT: [[VAL1:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; UNALIGNED-NEXT: [[VAL01:%.*]] = bitcast b32 [[VAL1]] to i32
+; UNALIGNED-NEXT: [[VAL13:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[VAL12:%.*]] = bitcast b32 [[VAL13]] to i32
; UNALIGNED-NEXT: [[ADD:%.*]] = add i32 [[VAL01]], [[VAL12]]
; UNALIGNED-NEXT: store i32 [[ADD]], ptr addrspace(1) [[OUT:%.*]], align 4
; UNALIGNED-NEXT: ret void
@@ -145,7 +153,7 @@ define amdgpu_kernel void @store_unknown_offset_align1_i8(ptr addrspace(1) noali
; UNALIGNED-LABEL: @store_unknown_offset_align1_i8(
; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i8], align 1, addrspace(5)
; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i8], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]]
-; UNALIGNED-NEXT: store <2 x i8> <i8 9, i8 10>, ptr addrspace(5) [[PTR0]], align 1
+; UNALIGNED-NEXT: store <2 x b8> <b8 9, b8 10>, ptr addrspace(5) [[PTR0]], align 1
; UNALIGNED-NEXT: ret void
;
%alloca = alloca [128 x i8], align 1, addrspace(5)
@@ -168,7 +176,7 @@ define amdgpu_kernel void @store_unknown_offset_align1_i16(ptr addrspace(1) noal
; UNALIGNED-LABEL: @store_unknown_offset_align1_i16(
; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i16], align 1, addrspace(5)
; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i16], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]]
-; UNALIGNED-NEXT: store <2 x i16> <i16 9, i16 10>, ptr addrspace(5) [[PTR0]], align 1
+; UNALIGNED-NEXT: store <2 x b16> <b16 9, b16 10>, ptr addrspace(5) [[PTR0]], align 1
; UNALIGNED-NEXT: ret void
;
%alloca = alloca [128 x i16], align 1, addrspace(5)
@@ -194,7 +202,7 @@ define amdgpu_kernel void @store_unknown_offset_align1_i32(ptr addrspace(1) noal
; UNALIGNED-LABEL: @store_unknown_offset_align1_i32(
; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i32], align 1, addrspace(5)
; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]]
-; UNALIGNED-NEXT: store <2 x i32> <i32 9, i32 10>, ptr addrspace(5) [[PTR0]], align 1
+; UNALIGNED-NEXT: store <2 x b32> <b32 9, b32 10>, ptr addrspace(5) [[PTR0]], align 1
; UNALIGNED-NEXT: ret void
;
%alloca = alloca [128 x i32], align 1, addrspace(5)
@@ -208,7 +216,7 @@ define amdgpu_kernel void @store_unknown_offset_align1_i32(ptr addrspace(1) noal
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32() {
; CHECK-LABEL: @merge_private_store_4_vector_elts_loads_v4i32(
; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [8 x i32], align 4, addrspace(5)
-; CHECK-NEXT: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, ptr addrspace(5) [[ALLOCA]], align 4
+; CHECK-NEXT: store <4 x b32> <b32 9, b32 1, b32 23, b32 19>, ptr addrspace(5) [[ALLOCA]], align 4
; CHECK-NEXT: ret void
;
%alloca = alloca [8 x i32], align 1, addrspace(5)
@@ -226,7 +234,7 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32() {
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8() {
; CHECK-LABEL: @merge_private_store_4_vector_elts_loads_v4i8(
; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [8 x i8], align 4, addrspace(5)
-; CHECK-NEXT: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, ptr addrspace(5) [[ALLOCA]], align 4
+; CHECK-NEXT: store <4 x b8> <b8 9, b8 1, b8 23, b8 19>, ptr addrspace(5) [[ALLOCA]], align 4
; CHECK-NEXT: ret void
;
%alloca = alloca [8 x i8], align 1, addrspace(5)
@@ -244,11 +252,15 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8() {
define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() {
; CHECK-LABEL: @merge_private_load_4_vector_elts_loads_v4i32(
; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [8 x i32], align 4, addrspace(5)
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr addrspace(5) [[ALLOCA]], align 4
-; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[LOAD34:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(5) [[ALLOCA]], align 4
+; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LOAD01]] to i32
+; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LOAD12]] to i32
+; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[LOAD23]] to i32
+; CHECK-NEXT: [[LOAD34:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LOAD34]] to i32
; CHECK-NEXT: ret void
;
%alloca = alloca [8 x i32], align 1, addrspace(5)
@@ -266,11 +278,15 @@ define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i32() {
define amdgpu_kernel void @merge_private_load_4_vector_elts_loads_v4i8() {
; CHECK-LABEL: @merge_private_load_4_vector_elts_loads_v4i8(
; CHECK-NEXT: [[ALLOCA:%.*]] = alloca [8 x i8], align 4, addrspace(5)
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(5) [[ALLOCA]], align 4
-; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[LOAD34:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b8>, ptr addrspace(5) [[ALLOCA]], align 4
+; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <4 x b8> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[LOAD01]] to i8
+; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <4 x b8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[LOAD12]] to i8
+; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <4 x b8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b8 [[LOAD23]] to i8
+; CHECK-NEXT: [[LOAD34:%.*]] = extractelement <4 x b8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b8 [[LOAD34]] to i8
; CHECK-NEXT: ret void
;
%alloca = alloca [8 x i8], align 1, addrspace(5)
@@ -294,7 +310,7 @@ define void @private_store_2xi16_align2_not_alloca(ptr addrspace(5) %p, ptr addr
; ALIGNED-NEXT: ret void
;
; UNALIGNED-LABEL: @private_store_2xi16_align2_not_alloca(
-; UNALIGNED-NEXT: store <2 x i16> <i16 1, i16 2>, ptr addrspace(5) [[R:%.*]], align 2
+; UNALIGNED-NEXT: store <2 x b16> <b16 1, b16 2>, ptr addrspace(5) [[R:%.*]], align 2
; UNALIGNED-NEXT: ret void
;
%gep.r = getelementptr i16, ptr addrspace(5) %r, i32 1
@@ -311,7 +327,7 @@ define void @private_store_2xi16_align1_not_alloca(ptr addrspace(5) %p, ptr addr
; ALIGNED-NEXT: ret void
;
; UNALIGNED-LABEL: @private_store_2xi16_align1_not_alloca(
-; UNALIGNED-NEXT: store <2 x i16> <i16 1, i16 2>, ptr addrspace(5) [[R:%.*]], align 1
+; UNALIGNED-NEXT: store <2 x b16> <b16 1, b16 2>, ptr addrspace(5) [[R:%.*]], align 1
; UNALIGNED-NEXT: ret void
;
%gep.r = getelementptr i16, ptr addrspace(5) %r, i32 1
@@ -332,9 +348,11 @@ define i32 @private_load_2xi16_align2_not_alloca(ptr addrspace(5) %p) #0 {
; ALIGNED-NEXT: ret i32 [[OR]]
;
; UNALIGNED-LABEL: @private_load_2xi16_align2_not_alloca(
-; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i16>, ptr addrspace(5) [[P:%.*]], align 2
-; UNALIGNED-NEXT: [[P_01:%.*]] = extractelement <2 x i16> [[TMP1]], i32 0
-; UNALIGNED-NEXT: [[P_12:%.*]] = extractelement <2 x i16> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x b16>, ptr addrspace(5) [[P:%.*]], align 2
+; UNALIGNED-NEXT: [[P_1:%.*]] = extractelement <2 x b16> [[TMP1]], i32 0
+; UNALIGNED-NEXT: [[P_01:%.*]] = bitcast b16 [[P_1]] to i16
+; UNALIGNED-NEXT: [[P_13:%.*]] = extractelement <2 x b16> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[P_12:%.*]] = bitcast b16 [[P_13]] to i16
; UNALIGNED-NEXT: [[ZEXT_0:%.*]] = zext i16 [[P_01]] to i32
; UNALIGNED-NEXT: [[ZEXT_1:%.*]] = zext i16 [[P_12]] to i32
; UNALIGNED-NEXT: [[SHL_1:%.*]] = shl i32 [[ZEXT_1]], 16
@@ -363,9 +381,11 @@ define i32 @private_load_2xi16_align1_not_alloca(ptr addrspace(5) %p) #0 {
; ALIGNED-NEXT: ret i32 [[OR]]
;
; UNALIGNED-LABEL: @private_load_2xi16_align1_not_alloca(
-; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i16>, ptr addrspace(5) [[P:%.*]], align 1
-; UNALIGNED-NEXT: [[P_01:%.*]] = extractelement <2 x i16> [[TMP1]], i32 0
-; UNALIGNED-NEXT: [[P_12:%.*]] = extractelement <2 x i16> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x b16>, ptr addrspace(5) [[P:%.*]], align 1
+; UNALIGNED-NEXT: [[P_1:%.*]] = extractelement <2 x b16> [[TMP1]], i32 0
+; UNALIGNED-NEXT: [[P_01:%.*]] = bitcast b16 [[P_1]] to i16
+; UNALIGNED-NEXT: [[P_13:%.*]] = extractelement <2 x b16> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[P_12:%.*]] = bitcast b16 [[P_13]] to i16
; UNALIGNED-NEXT: [[ZEXT_0:%.*]] = zext i16 [[P_01]] to i32
; UNALIGNED-NEXT: [[ZEXT_1:%.*]] = zext i16 [[P_12]] to i32
; UNALIGNED-NEXT: [[SHL_1:%.*]] = shl i32 [[ZEXT_1]], 16
@@ -396,9 +416,11 @@ define void @load_alloca16_unknown_offset_align1_i8(ptr addrspace(1) noalias %ou
; UNALIGNED-LABEL: @load_alloca16_unknown_offset_align1_i8(
; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i8], align 16, addrspace(5)
; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i8], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]]
-; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr addrspace(5) [[PTR0]], align 1
-; UNALIGNED-NEXT: [[VAL01:%.*]] = extractelement <2 x i8> [[TMP1]], i32 0
-; UNALIGNED-NEXT: [[VAL12:%.*]] = extractelement <2 x i8> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[TMP1:%.*]] = load <2 x b8>, ptr addrspace(5) [[PTR0]], align 1
+; UNALIGNED-NEXT: [[VAL1:%.*]] = extractelement <2 x b8> [[TMP1]], i32 0
+; UNALIGNED-NEXT: [[VAL01:%.*]] = bitcast b8 [[VAL1]] to i8
+; UNALIGNED-NEXT: [[VAL13:%.*]] = extractelement <2 x b8> [[TMP1]], i32 1
+; UNALIGNED-NEXT: [[VAL12:%.*]] = bitcast b8 [[VAL13]] to i8
; UNALIGNED-NEXT: [[ADD:%.*]] = add i8 [[VAL01]], [[VAL12]]
; UNALIGNED-NEXT: store i8 [[ADD]], ptr addrspace(1) [[OUT:%.*]], align 1
; UNALIGNED-NEXT: ret void
@@ -425,7 +447,7 @@ define void @store_alloca16_unknown_offset_align1_i32(ptr addrspace(1) noalias %
; UNALIGNED-LABEL: @store_alloca16_unknown_offset_align1_i32(
; UNALIGNED-NEXT: [[ALLOCA:%.*]] = alloca [128 x i32], align 16, addrspace(5)
; UNALIGNED-NEXT: [[PTR0:%.*]] = getelementptr inbounds [128 x i32], ptr addrspace(5) [[ALLOCA]], i32 0, i32 [[OFFSET:%.*]]
-; UNALIGNED-NEXT: store <2 x i32> <i32 9, i32 10>, ptr addrspace(5) [[PTR0]], align 4
+; UNALIGNED-NEXT: store <2 x b32> <b32 9, b32 10>, ptr addrspace(5) [[PTR0]], align 4
; UNALIGNED-NEXT: ret void
;
%alloca = alloca [128 x i32], align 16, addrspace(5)
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
index b558c0996100e..aec2cb00d6cf3 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/complex-index.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
@@ -7,10 +8,49 @@ declare i64 @_Z12get_group_idj(i32)
declare double @llvm.fmuladd.f64(double, double, double)
-; CHECK-LABEL: @factorizedVsNonfactorizedAccess(
-; CHECK: load <2 x float>
-; CHECK: store <2 x float>
define amdgpu_kernel void @factorizedVsNonfactorizedAccess(ptr addrspace(1) nocapture %c) {
+; CHECK-LABEL: define amdgpu_kernel void @factorizedVsNonfactorizedAccess(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[CALL:%.*]] = tail call i64 @_Z12get_local_idj(i32 0)
+; CHECK-NEXT: [[CALL1:%.*]] = tail call i64 @_Z12get_group_idj(i32 0)
+; CHECK-NEXT: [[DIV:%.*]] = lshr i64 [[CALL]], 4
+; CHECK-NEXT: [[DIV2:%.*]] = lshr i64 [[CALL1]], 3
+; CHECK-NEXT: [[MUL:%.*]] = shl i64 [[DIV2]], 7
+; CHECK-NEXT: [[REM:%.*]] = shl i64 [[CALL]], 3
+; CHECK-NEXT: [[MUL3:%.*]] = and i64 [[REM]], 120
+; CHECK-NEXT: [[ADD:%.*]] = or i64 [[MUL]], [[MUL3]]
+; CHECK-NEXT: [[REM4:%.*]] = shl i64 [[CALL1]], 7
+; CHECK-NEXT: [[MUL5:%.*]] = and i64 [[REM4]], 896
+; CHECK-NEXT: [[MUL6:%.*]] = shl nuw nsw i64 [[DIV]], 3
+; CHECK-NEXT: [[ADD7:%.*]] = add nuw i64 [[MUL5]], [[MUL6]]
+; CHECK-NEXT: [[MUL9:%.*]] = shl i64 [[ADD7]], 10
+; CHECK-NEXT: [[ADD10:%.*]] = add i64 [[MUL9]], [[ADD]]
+; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[C]], i64 [[ADD10]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[ARRAYIDX]], align 4
+; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LOAD11]] to float
+; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LOAD22]] to float
+; CHECK-NEXT: [[CONV:%.*]] = fpext float [[TMP1]] to double
+; CHECK-NEXT: [[MUL11:%.*]] = fmul double [[CONV]], f0x3FEAB481D8F35506
+; CHECK-NEXT: [[CONV12:%.*]] = fptrunc double [[MUL11]] to float
+; CHECK-NEXT: [[CONV18:%.*]] = fpext float [[CONV12]] to double
+; CHECK-NEXT: [[STOREVAL1:%.*]] = tail call double @llvm.fmuladd.f64(double f0x3FF4FFAFBBEC946A, double 0.000000e+00, double [[CONV18]])
+; CHECK-NEXT: [[CSTOREVAL1:%.*]] = fptrunc double [[STOREVAL1]] to float
+; CHECK-NEXT: [[CONV25:%.*]] = fpext float [[TMP2]] to double
+; CHECK-NEXT: [[MUL26:%.*]] = fmul double [[CONV25]], f0x3FEAB481D8F35506
+; CHECK-NEXT: [[CONV27:%.*]] = fptrunc double [[MUL26]] to float
+; CHECK-NEXT: [[CONV34:%.*]] = fpext float [[CONV27]] to double
+; CHECK-NEXT: [[STOREVAL2:%.*]] = tail call double @llvm.fmuladd.f64(double f0x3FF4FFAFBBEC946A, double 0.000000e+00, double [[CONV34]])
+; CHECK-NEXT: [[CSTOREVAL2:%.*]] = fptrunc double [[STOREVAL2]] to float
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float [[CSTOREVAL1]] to b32
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x b32> poison, b32 [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast float [[CSTOREVAL2]] to b32
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x b32> [[TMP4]], b32 [[TMP5]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP6]], ptr addrspace(1) [[ARRAYIDX]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%call = tail call i64 @_Z12get_local_idj(i32 0)
%call1 = tail call i64 @_Z12get_group_idj(i32 0)
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/dont-adjust-globalobj-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/dont-adjust-globalobj-alignment.ll
index 4486368290eaf..1b1b95ab458a8 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/dont-adjust-globalobj-alignment.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/dont-adjust-globalobj-alignment.ll
@@ -1,4 +1,5 @@
-; RUN: opt -S -passes=load-store-vectorizer --mcpu=hawaii -mattr=+unaligned-access-mode,+unaligned-scratch-access,+max-private-element-size-16 < %s | FileCheck --match-full-lines %s
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=load-store-vectorizer --mcpu=hawaii -mattr=+unaligned-access-mode,+unaligned-scratch-access,+max-private-element-size-16 < %s | FileCheck %s
target triple = "amdgcn--"
@@ -7,12 +8,12 @@ target triple = "amdgcn--"
; Verify that the alignment of the global remains at 1, even if we vectorize
; the stores.
;
-; CHECK: @G = internal addrspace(5) global [8 x i16] undef, align 1
define void @private_store_2xi16_align2_not_alloca(ptr addrspace(5) %p, ptr addrspace(5) %r) {
-; CHECK: define void @private_store_2xi16_align2_not_alloca(ptr addrspace(5) [[P:%.*]], ptr addrspace(5) [[R:%.*]]) #0 {
+; CHECK-LABEL: define void @private_store_2xi16_align2_not_alloca(
+; CHECK-SAME: ptr addrspace(5) [[P:%.*]], ptr addrspace(5) [[R:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i16, ptr addrspace(5) @G, i32 0
-; CHECK-NEXT: store <2 x i16> <i16 1, i16 2>, ptr addrspace(5) [[GEP0]], align 1
+; CHECK-NEXT: store <2 x b16> <b16 1, b16 2>, ptr addrspace(5) [[GEP0]], align 1
; CHECK-NEXT: ret void
;
%gep0 = getelementptr i16, ptr addrspace(5) @G, i32 0
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
index 009c3b708d0c1..641534065cf96 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/extended-index.ll
@@ -1,13 +1,27 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
declare i32 @llvm.amdgcn.workitem.id.x() #1
-; CHECK-LABEL: @basic_merge_sext_index(
-; CHECK: sext i32 %id.x to i64
-; CHECK: load <2 x float>
-; CHECK: store <2 x float> zeroinitializer
define amdgpu_kernel void @basic_merge_sext_index(ptr addrspace(1) nocapture %a, ptr addrspace(1) nocapture %b, ptr addrspace(1) nocapture readonly %c) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @basic_merge_sext_index(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(1) captures(none) [[B:%.*]], ptr addrspace(1) readonly captures(none) [[C:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[SEXT_ID_X:%.*]] = sext i32 [[ID_X]] to i64
+; CHECK-NEXT: [[A_IDX_X:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 [[SEXT_ID_X]]
+; CHECK-NEXT: [[C_IDX_X:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[C]], i64 [[SEXT_ID_X]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[C_IDX_X]], align 4
+; CHECK-NEXT: [[LD_C1:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_C1]] to float
+; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_C_IDX_12]] to float
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(1) [[A_IDX_X]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: store float [[ADD]], ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%id.x = call i32 @llvm.amdgcn.workitem.id.x()
%sext.id.x = sext i32 %id.x to i64
@@ -27,11 +41,24 @@ entry:
ret void
}
-; CHECK-LABEL: @basic_merge_zext_index(
-; CHECK: zext i32 %id.x to i64
-; CHECK: load <2 x float>
-; CHECK: store <2 x float>
define amdgpu_kernel void @basic_merge_zext_index(ptr addrspace(1) nocapture %a, ptr addrspace(1) nocapture %b, ptr addrspace(1) nocapture readonly %c) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @basic_merge_zext_index(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(1) captures(none) [[B:%.*]], ptr addrspace(1) readonly captures(none) [[C:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[ZEXT_ID_X:%.*]] = zext i32 [[ID_X]] to i64
+; CHECK-NEXT: [[A_IDX_X:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 [[ZEXT_ID_X]]
+; CHECK-NEXT: [[C_IDX_X:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[C]], i64 [[ZEXT_ID_X]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[C_IDX_X]], align 4
+; CHECK-NEXT: [[LD_C1:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_C1]] to float
+; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_C_IDX_12]] to float
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(1) [[A_IDX_X]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: store float [[ADD]], ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%id.x = call i32 @llvm.amdgcn.workitem.id.x()
%zext.id.x = zext i32 %id.x to i64
@@ -50,10 +77,25 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_op_zext_index(
-; CHECK: load <2 x float>
-; CHECK: store <2 x float>
define amdgpu_kernel void @merge_op_zext_index(ptr addrspace(1) nocapture noalias %a, ptr addrspace(1) nocapture noalias %b, ptr addrspace(1) nocapture readonly noalias %c) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_op_zext_index(
+; CHECK-SAME: ptr addrspace(1) noalias captures(none) [[A:%.*]], ptr addrspace(1) noalias captures(none) [[B:%.*]], ptr addrspace(1) noalias readonly captures(none) [[C:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[SHL:%.*]] = shl i32 [[ID_X]], 2
+; CHECK-NEXT: [[ZEXT_ID_X:%.*]] = zext i32 [[SHL]] to i64
+; CHECK-NEXT: [[A_0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 [[ZEXT_ID_X]]
+; CHECK-NEXT: [[C_0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[C]], i64 [[ZEXT_ID_X]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[C_0]], align 4
+; CHECK-NEXT: [[LD_C_01:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_C_01]] to float
+; CHECK-NEXT: [[LD_C_12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_C_12]] to float
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(1) [[A_0]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: store float [[ADD]], ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%id.x = call i32 @llvm.amdgcn.workitem.id.x()
%shl = shl i32 %id.x, 2
@@ -77,10 +119,25 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_op_sext_index(
-; CHECK: load <2 x float>
-; CHECK: store <2 x float>
define amdgpu_kernel void @merge_op_sext_index(ptr addrspace(1) nocapture noalias %a, ptr addrspace(1) nocapture noalias %b, ptr addrspace(1) nocapture readonly noalias %c) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_op_sext_index(
+; CHECK-SAME: ptr addrspace(1) noalias captures(none) [[A:%.*]], ptr addrspace(1) noalias captures(none) [[B:%.*]], ptr addrspace(1) noalias readonly captures(none) [[C:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ID_X:%.*]] = call i32 @llvm.amdgcn.workitem.id.x()
+; CHECK-NEXT: [[SHL:%.*]] = shl i32 [[ID_X]], 2
+; CHECK-NEXT: [[ZEXT_ID_X:%.*]] = sext i32 [[SHL]] to i64
+; CHECK-NEXT: [[A_0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 [[ZEXT_ID_X]]
+; CHECK-NEXT: [[C_0:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[C]], i64 [[ZEXT_ID_X]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[C_0]], align 4
+; CHECK-NEXT: [[LD_C_01:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_C_01]] to float
+; CHECK-NEXT: [[LD_C_12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_C_12]] to float
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(1) [[A_0]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: store float [[ADD]], ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%id.x = call i32 @llvm.amdgcn.workitem.id.x()
%shl = shl i32 %id.x, 2
@@ -107,11 +164,36 @@ entry:
; This case fails to vectorize if not using the extra extension
; handling in isConsecutiveAccess.
-; CHECK-LABEL: @zext_trunc_phi_1(
-; CHECK: loop:
-; CHECK: load <2 x i32>
-; CHECK: store <2 x i32>
define amdgpu_kernel void @zext_trunc_phi_1(ptr addrspace(1) nocapture noalias %a, ptr addrspace(1) nocapture noalias %b, ptr addrspace(1) nocapture readonly noalias %c, i32 %n, i64 %arst, i64 %aoeu) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @zext_trunc_phi_1(
+; CHECK-SAME: ptr addrspace(1) noalias captures(none) [[A:%.*]], ptr addrspace(1) noalias captures(none) [[B:%.*]], ptr addrspace(1) noalias readonly captures(none) [[C:%.*]], i32 [[N:%.*]], i64 [[ARST:%.*]], i64 [[AOEU:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[CMP0:%.*]] = icmp eq i32 [[N]], 0
+; CHECK-NEXT: br i1 [[CMP0]], label %[[EXIT:.*]], label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_NEXT:%.*]], %[[LOOP]] ], [ 0, %[[ENTRY]] ]
+; CHECK-NEXT: [[TRUNC_IV:%.*]] = trunc i64 [[INDVARS_IV]] to i32
+; CHECK-NEXT: [[IDX:%.*]] = shl i32 [[TRUNC_IV]], 4
+; CHECK-NEXT: [[IDX_EXT:%.*]] = zext i32 [[IDX]] to i64
+; CHECK-NEXT: [[C_0:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[C]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[A_0:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[A]], i64 [[IDX_EXT]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[C_0]], align 4
+; CHECK-NEXT: [[LD_C_01:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_C_01]] to i32
+; CHECK-NEXT: [[LD_C_12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_C_12]] to i32
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32 [[TMP1]] to b32
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x b32> poison, b32 [[TMP3]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x b32> [[TMP4]], b32 [[TMP5]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP6]], ptr addrspace(1) [[A_0]], align 4
+; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add i64 [[INDVARS_IV]], 1
+; CHECK-NEXT: [[LFTR_WIDEIV:%.*]] = trunc i64 [[INDVARS_IV_NEXT]] to i32
+; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[LFTR_WIDEIV]], [[N]]
+; CHECK-NEXT: br i1 [[EXITCOND]], label %[[EXIT]], label %[[LOOP]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
entry:
%cmp0 = icmp eq i32 %n, 0
br i1 %cmp0, label %exit, label %loop
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/fence-noalias-metadata.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/fence-noalias-metadata.ll
index d2f19efcaf160..2cfda50144740 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/fence-noalias-metadata.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/fence-noalias-metadata.ll
@@ -15,9 +15,11 @@
define void @vectorize_loads_across_fence_with_noalias(ptr addrspace(1) %ptr) {
; CHECK-LABEL: define void @vectorize_loads_across_fence_with_noalias(
; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[PTR]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
-; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(1) [[PTR]], align 8, !alias.scope [[META0:![0-9]+]], !noalias [[META3:![0-9]+]]
+; CHECK-NEXT: [[LOAD1:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[LOAD01:%.*]] = bitcast b32 [[LOAD1]] to i32
+; CHECK-NEXT: [[LOAD13:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[LOAD12:%.*]] = bitcast b32 [[LOAD13]] to i32
; CHECK-NEXT: fence syncscope("workgroup") release, !noalias [[META5:![0-9]+]]
; CHECK-NEXT: fence syncscope("workgroup") acquire, !noalias [[META5]]
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[LOAD01]], [[LOAD12]]
@@ -39,9 +41,11 @@ define void @vectorize_loads_across_fence_with_noalias(ptr addrspace(1) %ptr) {
define void @vectorize_loads_across_fence_with_alias_scope(ptr addrspace(1) %ptr) {
; CHECK-LABEL: define void @vectorize_loads_across_fence_with_alias_scope(
; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[PTR]], align 8, !alias.scope [[META0]], !noalias [[META6:![0-9]+]]
-; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(1) [[PTR]], align 8, !alias.scope [[META0]], !noalias [[META6:![0-9]+]]
+; CHECK-NEXT: [[LOAD1:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[LOAD01:%.*]] = bitcast b32 [[LOAD1]] to i32
+; CHECK-NEXT: [[LOAD13:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[LOAD12:%.*]] = bitcast b32 [[LOAD13]] to i32
; CHECK-NEXT: fence syncscope("workgroup") release, !alias.scope [[META6]]
; CHECK-NEXT: fence syncscope("workgroup") acquire, !alias.scope [[META6]]
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[LOAD01]], [[LOAD12]]
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll
index 807ef651dbb32..fefd0bb79a18f 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/gep-bitcast.ll
@@ -1,10 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S -mtriple=amdgcn--amdhsa -passes=load-store-vectorizer < %s | FileCheck %s
; RUN: opt -S -mtriple=amdgcn--amdhsa -passes='function(load-store-vectorizer)' < %s | FileCheck %s
; Check that vectorizer can find a GEP through bitcast
-; CHECK-LABEL: @vect_zext_bitcast_f32_to_i32_idx
-; CHECK: load <4 x i32>
define void @vect_zext_bitcast_f32_to_i32_idx(ptr addrspace(1) %arg1, i32 %base) {
+; CHECK-LABEL: define void @vect_zext_bitcast_f32_to_i32_idx(
+; CHECK-SAME: ptr addrspace(1) [[ARG1:%.*]], i32 [[BASE:%.*]]) {
+; CHECK-NEXT: [[ADD1:%.*]] = add nuw i32 [[BASE]], 0
+; CHECK-NEXT: [[ZEXT1:%.*]] = zext i32 [[ADD1]] to i64
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[ARG1]], i64 [[ZEXT1]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LOAD11]] to i32
+; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LOAD22]] to i32
+; CHECK-NEXT: [[LOAD33:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[LOAD33]] to i32
+; CHECK-NEXT: [[LOAD44:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LOAD44]] to i32
+; CHECK-NEXT: ret void
+;
%add1 = add nuw i32 %base, 0
%zext1 = zext i32 %add1 to i64
%gep1 = getelementptr inbounds float, ptr addrspace(1) %arg1, i64 %zext1
@@ -24,12 +39,27 @@ define void @vect_zext_bitcast_f32_to_i32_idx(ptr addrspace(1) %arg1, i32 %base)
ret void
}
-; CHECK-LABEL: @vect_zext_bitcast_i8_st1_to_i32_idx
-; CHECK: load i32
-; CHECK: load i32
-; CHECK: load i32
-; CHECK: load i32
define void @vect_zext_bitcast_i8_st1_to_i32_idx(ptr addrspace(1) %arg1, i32 %base) {
+; CHECK-LABEL: define void @vect_zext_bitcast_i8_st1_to_i32_idx(
+; CHECK-SAME: ptr addrspace(1) [[ARG1:%.*]], i32 [[BASE:%.*]]) {
+; CHECK-NEXT: [[ADD1:%.*]] = add nuw i32 [[BASE]], 0
+; CHECK-NEXT: [[ZEXT1:%.*]] = zext i32 [[ADD1]] to i64
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ARG1]], i64 [[ZEXT1]]
+; CHECK-NEXT: [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT: [[ADD2:%.*]] = add nuw i32 [[BASE]], 1
+; CHECK-NEXT: [[ZEXT2:%.*]] = zext i32 [[ADD2]] to i64
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ARG1]], i64 [[ZEXT2]]
+; CHECK-NEXT: [[LOAD2:%.*]] = load i32, ptr addrspace(1) [[GEP2]], align 4
+; CHECK-NEXT: [[ADD3:%.*]] = add nuw i32 [[BASE]], 2
+; CHECK-NEXT: [[ZEXT3:%.*]] = zext i32 [[ADD3]] to i64
+; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ARG1]], i64 [[ZEXT3]]
+; CHECK-NEXT: [[LOAD3:%.*]] = load i32, ptr addrspace(1) [[GEP3]], align 4
+; CHECK-NEXT: [[ADD4:%.*]] = add nuw i32 [[BASE]], 3
+; CHECK-NEXT: [[ZEXT4:%.*]] = zext i32 [[ADD4]] to i64
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ARG1]], i64 [[ZEXT4]]
+; CHECK-NEXT: [[LOAD4:%.*]] = load i32, ptr addrspace(1) [[GEP4]], align 4
+; CHECK-NEXT: ret void
+;
%add1 = add nuw i32 %base, 0
%zext1 = zext i32 %add1 to i64
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %arg1, i64 %zext1
@@ -49,9 +79,23 @@ define void @vect_zext_bitcast_i8_st1_to_i32_idx(ptr addrspace(1) %arg1, i32 %ba
ret void
}
-; CHECK-LABEL: @vect_zext_bitcast_i8_st4_to_i32_idx
-; CHECK: load <4 x i32>
define void @vect_zext_bitcast_i8_st4_to_i32_idx(ptr addrspace(1) %arg1, i32 %base) {
+; CHECK-LABEL: define void @vect_zext_bitcast_i8_st4_to_i32_idx(
+; CHECK-SAME: ptr addrspace(1) [[ARG1:%.*]], i32 [[BASE:%.*]]) {
+; CHECK-NEXT: [[ADD1:%.*]] = add nuw i32 [[BASE]], 0
+; CHECK-NEXT: [[ZEXT1:%.*]] = zext i32 [[ADD1]] to i64
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ARG1]], i64 [[ZEXT1]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LOAD11]] to i32
+; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LOAD22]] to i32
+; CHECK-NEXT: [[LOAD33:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[LOAD33]] to i32
+; CHECK-NEXT: [[LOAD44:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LOAD44]] to i32
+; CHECK-NEXT: ret void
+;
%add1 = add nuw i32 %base, 0
%zext1 = zext i32 %add1 to i64
%gep1 = getelementptr inbounds i8, ptr addrspace(1) %arg1, i64 %zext1
@@ -71,9 +115,19 @@ define void @vect_zext_bitcast_i8_st4_to_i32_idx(ptr addrspace(1) %arg1, i32 %ba
ret void
}
-; CHECK-LABEL: @vect_zext_bitcast_negative_ptr_delta
-; CHECK: load <2 x i32>
define void @vect_zext_bitcast_negative_ptr_delta(ptr addrspace(1) %p, i32 %base) {
+; CHECK-LABEL: define void @vect_zext_bitcast_negative_ptr_delta(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[BASE:%.*]]) {
+; CHECK-NEXT: [[A_OFFSET:%.*]] = add nuw i32 [[BASE]], 4
+; CHECK-NEXT: [[A_OFFSET_ZEXTED:%.*]] = zext i32 [[A_OFFSET]] to i64
+; CHECK-NEXT: [[A_PTR:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[P]], i64 [[A_OFFSET_ZEXTED]]
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(1) [[A_PTR]], align 4
+; CHECK-NEXT: [[A_VAL1:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[A_VAL1]] to i32
+; CHECK-NEXT: [[B_VAL2:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[B_VAL2]] to i32
+; CHECK-NEXT: ret void
+;
%a.offset = add nuw i32 %base, 4
%t.offset.zexted = zext i32 %base to i64
%a.offset.zexted = zext i32 %a.offset to i64
@@ -86,10 +140,19 @@ define void @vect_zext_bitcast_negative_ptr_delta(ptr addrspace(1) %p, i32 %base
}
; Check i1 corner case
-; CHECK-LABEL: @zexted_i1_gep_index
-; CHECK: load i32
-; CHECK: load i32
define void @zexted_i1_gep_index(ptr addrspace(1) %p, i32 %val) {
+; CHECK-LABEL: define void @zexted_i1_gep_index(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[SELECTOR:%.*]] = icmp eq i32 [[VAL]], 0
+; CHECK-NEXT: [[FLIPPED:%.*]] = xor i1 [[SELECTOR]], true
+; CHECK-NEXT: [[INDEX_0:%.*]] = zext i1 [[SELECTOR]] to i64
+; CHECK-NEXT: [[INDEX_1:%.*]] = zext i1 [[FLIPPED]] to i64
+; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P]], i64 [[INDEX_0]]
+; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P]], i64 [[INDEX_1]]
+; CHECK-NEXT: [[VAL0:%.*]] = load i32, ptr addrspace(1) [[GEP_0]], align 4
+; CHECK-NEXT: [[VAL1:%.*]] = load i32, ptr addrspace(1) [[GEP_1]], align 4
+; CHECK-NEXT: ret void
+;
%selector = icmp eq i32 %val, 0
%flipped = xor i1 %selector, 1
%index.0 = zext i1 %selector to i64
@@ -102,10 +165,19 @@ define void @zexted_i1_gep_index(ptr addrspace(1) %p, i32 %val) {
}
; Check i1 corner case
-; CHECK-LABEL: @sexted_i1_gep_index
-; CHECK: load i32
-; CHECK: load i32
define void @sexted_i1_gep_index(ptr addrspace(1) %p, i32 %val) {
+; CHECK-LABEL: define void @sexted_i1_gep_index(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[SELECTOR:%.*]] = icmp eq i32 [[VAL]], 0
+; CHECK-NEXT: [[FLIPPED:%.*]] = xor i1 [[SELECTOR]], true
+; CHECK-NEXT: [[INDEX_0:%.*]] = sext i1 [[SELECTOR]] to i64
+; CHECK-NEXT: [[INDEX_1:%.*]] = sext i1 [[FLIPPED]] to i64
+; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P]], i64 [[INDEX_0]]
+; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P]], i64 [[INDEX_1]]
+; CHECK-NEXT: [[VAL0:%.*]] = load i32, ptr addrspace(1) [[GEP_0]], align 4
+; CHECK-NEXT: [[VAL1:%.*]] = load i32, ptr addrspace(1) [[GEP_1]], align 4
+; CHECK-NEXT: ret void
+;
%selector = icmp eq i32 %val, 0
%flipped = xor i1 %selector, 1
%index.0 = sext i1 %selector to i64
@@ -117,10 +189,22 @@ define void @sexted_i1_gep_index(ptr addrspace(1) %p, i32 %val) {
ret void
}
-; CHECK-LABEL: @zexted_i1_gep_index_different_bbs
-; CHECK: load i32
-; CHECK: load i32
define void @zexted_i1_gep_index_different_bbs(ptr addrspace(1) %p, i32 %val) {
+; CHECK-LABEL: define void @zexted_i1_gep_index_different_bbs(
+; CHECK-SAME: ptr addrspace(1) [[P:%.*]], i32 [[VAL:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SELECTOR:%.*]] = icmp eq i32 [[VAL]], 0
+; CHECK-NEXT: [[FLIPPED:%.*]] = xor i1 [[SELECTOR]], true
+; CHECK-NEXT: [[INDEX_0:%.*]] = zext i1 [[SELECTOR]] to i64
+; CHECK-NEXT: [[INDEX_1:%.*]] = zext i1 [[FLIPPED]] to i64
+; CHECK-NEXT: [[GEP_0:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P]], i64 [[INDEX_0]]
+; CHECK-NEXT: br label %[[NEXT:.*]]
+; CHECK: [[NEXT]]:
+; CHECK-NEXT: [[GEP_1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[P]], i64 [[INDEX_1]]
+; CHECK-NEXT: [[VAL0:%.*]] = load i32, ptr addrspace(1) [[GEP_0]], align 4
+; CHECK-NEXT: [[VAL1:%.*]] = load i32, ptr addrspace(1) [[GEP_1]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%selector = icmp eq i32 %val, 0
%flipped = xor i1 %selector, 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
index 7e8202b9de37c..b7954cf60b09f 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/insertion-point.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
@@ -5,12 +6,26 @@
; inserted at the position of the first load in the chain, and stores should be
; inserted at the position of the last store.
-; CHECK-LABEL: @insert_load_point(
-; CHECK: %z = add i32 %x, 4
-; CHECK: load <2 x float>
-; CHECK: %w = add i32 %y, 9
-; CHECK: %foo = add i32 %z, %w
define amdgpu_kernel void @insert_load_point(ptr addrspace(1) nocapture %a, ptr addrspace(1) nocapture %b, ptr addrspace(1) nocapture readonly %c, i64 %idx, i32 %x, i32 %y) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @insert_load_point(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(1) captures(none) [[B:%.*]], ptr addrspace(1) readonly captures(none) [[C:%.*]], i64 [[IDX:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_IDX_X:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 [[IDX]]
+; CHECK-NEXT: [[C_IDX_X:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[C]], i64 [[IDX]]
+; CHECK-NEXT: [[Z:%.*]] = add i32 [[X]], 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[C_IDX_X]], align 4
+; CHECK-NEXT: [[LD_C1:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_C1]] to float
+; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_C_IDX_12]] to float
+; CHECK-NEXT: [[W:%.*]] = add i32 [[Y]], 9
+; CHECK-NEXT: [[FOO:%.*]] = add i32 [[Z]], [[W]]
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(1) [[A_IDX_X]], align 4
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: store float [[ADD]], ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: store i32 [[FOO]], ptr addrspace(3) null, align 4
+; CHECK-NEXT: ret void
+;
entry:
%a.idx.x = getelementptr inbounds float, ptr addrspace(1) %a, i64 %idx
%c.idx.x = getelementptr inbounds float, ptr addrspace(1) %c, i64 %idx
@@ -32,12 +47,26 @@ entry:
ret void
}
-; CHECK-LABEL: @insert_store_point(
-; CHECK: %z = add i32 %x, 4
-; CHECK: %w = add i32 %y, 9
-; CHECK: store <2 x float>
-; CHECK: %foo = add i32 %z, %w
define amdgpu_kernel void @insert_store_point(ptr addrspace(1) nocapture %a, ptr addrspace(1) nocapture %b, ptr addrspace(1) nocapture readonly %c, i64 %idx, i32 %x, i32 %y) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @insert_store_point(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(1) captures(none) [[B:%.*]], ptr addrspace(1) readonly captures(none) [[C:%.*]], i64 [[IDX:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_IDX_X:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 [[IDX]]
+; CHECK-NEXT: [[C_IDX_X:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[C]], i64 [[IDX]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[C_IDX_X]], align 4
+; CHECK-NEXT: [[LD_C1:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_C1]] to float
+; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_C_IDX_12]] to float
+; CHECK-NEXT: [[Z:%.*]] = add i32 [[X]], 4
+; CHECK-NEXT: [[W:%.*]] = add i32 [[Y]], 9
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(1) [[A_IDX_X]], align 4
+; CHECK-NEXT: [[FOO:%.*]] = add i32 [[Z]], [[W]]
+; CHECK-NEXT: [[ADD:%.*]] = fadd float [[TMP1]], [[TMP2]]
+; CHECK-NEXT: store float [[ADD]], ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: store i32 [[FOO]], ptr addrspace(3) null, align 4
+; CHECK-NEXT: ret void
+;
entry:
%a.idx.x = getelementptr inbounds float, ptr addrspace(1) %a, i64 %idx
%c.idx.x = getelementptr inbounds float, ptr addrspace(1) %c, i64 %idx
@@ -63,12 +92,18 @@ entry:
; vectorize the first three stores as <3 x float>, but this vectorized store must
; be inserted at the location of the third scalar store, not the fourth one.
;
-; CHECK-LABEL: @insert_store_point_alias
-; CHECK: store <3 x float>
-; CHECK: load float, ptr addrspace(1) %a.idx.2
-; CHECK: store float
-; CHECK-SAME: %a.idx.3
define float @insert_store_point_alias(ptr addrspace(1) nocapture %a, i64 %idx) {
+; CHECK-LABEL: define float @insert_store_point_alias(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT: [[A_IDX:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 [[IDX]]
+; CHECK-NEXT: [[A_IDX_1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A_IDX]], i64 1
+; CHECK-NEXT: [[A_IDX_2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A_IDX_1]], i64 1
+; CHECK-NEXT: [[A_IDX_3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A_IDX_2]], i64 1
+; CHECK-NEXT: store <3 x b32> zeroinitializer, ptr addrspace(1) [[A_IDX]], align 4
+; CHECK-NEXT: [[X:%.*]] = load float, ptr addrspace(1) [[A_IDX_2]], align 4
+; CHECK-NEXT: store float 0.000000e+00, ptr addrspace(1) [[A_IDX_3]], align 4
+; CHECK-NEXT: ret float [[X]]
+;
%a.idx = getelementptr inbounds float, ptr addrspace(1) %a, i64 %idx
%a.idx.1 = getelementptr inbounds float, ptr addrspace(1) %a.idx, i64 1
%a.idx.2 = getelementptr inbounds float, ptr addrspace(1) %a.idx.1, i64 1
@@ -87,11 +122,17 @@ define float @insert_store_point_alias(ptr addrspace(1) nocapture %a, i64 %idx)
; vectorize three of the stores before the load, but the important thing is that
; we *don't* sink the store to a[idx + 1] below the load.
;
-; CHECK-LABEL: @insert_store_point_alias_ooo
-; CHECK: store <3 x float>{{.*}} %a.idx.1
-; CHECK: load float, ptr addrspace(1) %a.idx.2
-; CHECK: store float{{.*}} %a.idx
define float @insert_store_point_alias_ooo(ptr addrspace(1) nocapture %a, i64 %idx) {
+; CHECK-LABEL: define float @insert_store_point_alias_ooo(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], i64 [[IDX:%.*]]) {
+; CHECK-NEXT: [[A_IDX:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A]], i64 [[IDX]]
+; CHECK-NEXT: [[A_IDX_1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A_IDX]], i64 1
+; CHECK-NEXT: [[A_IDX_2:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[A_IDX_1]], i64 1
+; CHECK-NEXT: store <3 x b32> zeroinitializer, ptr addrspace(1) [[A_IDX_1]], align 4
+; CHECK-NEXT: [[X:%.*]] = load float, ptr addrspace(1) [[A_IDX_2]], align 4
+; CHECK-NEXT: store float 0.000000e+00, ptr addrspace(1) [[A_IDX]], align 4
+; CHECK-NEXT: ret float [[X]]
+;
%a.idx = getelementptr inbounds float, ptr addrspace(1) %a, i64 %idx
%a.idx.1 = getelementptr inbounds float, ptr addrspace(1) %a.idx, i64 1
%a.idx.2 = getelementptr inbounds float, ptr addrspace(1) %a.idx.1, i64 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
index 0a09010dadc92..a0d4c2f397153 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll
@@ -1,11 +1,21 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; RUN: opt -mtriple=amdgcn-amd-amdhsa -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
-; CHECK-LABEL: @interleave
-; CHECK: load <2 x double>, ptr addrspace(1) %{{.}}, align 8{{$}}
-; CHECK: store <2 x double> zeroinitializer
-; CHECK: store double %add
define amdgpu_kernel void @interleave(ptr addrspace(1) nocapture %a, ptr addrspace(1) nocapture %b, ptr addrspace(1) nocapture readonly %c) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @interleave(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(1) captures(none) [[B:%.*]], ptr addrspace(1) readonly captures(none) [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b64>, ptr addrspace(1) [[C]], align 8
+; CHECK-NEXT: [[LD_C1:%.*]] = extractelement <2 x b64> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b64 [[LD_C1]] to double
+; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = extractelement <2 x b64> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[LD_C_IDX_12]] to double
+; CHECK-NEXT: store <2 x b64> zeroinitializer, ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: [[ADD:%.*]] = fadd double [[TMP1]], [[TMP2]]
+; CHECK-NEXT: store double [[ADD]], ptr addrspace(1) [[B]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%a.idx.1 = getelementptr inbounds double, ptr addrspace(1) %a, i64 1
%c.idx.1 = getelementptr inbounds double, ptr addrspace(1) %c, i64 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
index 3bbc7d6725962..8ecae7c79d34f 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores-private.ll
@@ -1,21 +1,38 @@
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-4,-unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT4,ELT4-ALIGNED,ALIGNED,ALL %s
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-8,-unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT8,ALIGNED,ALL %s
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-16,-unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT16,ALIGNED,ALL %s
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-4,+unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT4,ELT4-UNALIGNED,UNALIGNED,ALL %s
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-8,+unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT8,ELT8-UNALIGNED,UNALIGNED,ALL %s
-; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-16,+unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT16,ELT16-UNALIGNED,UNALIGNED,ALL %s
-
-; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32
-; ELT4-ALIGNED: store i32
-; ELT4-ALIGNED: store i32
-; ELT4-ALIGNED: store i32
-; ELT4-ALIGNED: store i32
-
-; ELT8: store <2 x i32>
-; ELT8: store <2 x i32>
-
-; ELT16-UNALIGNED: store <4 x i32>
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-4,-unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT4,ELT4-ALIGNED,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-8,-unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT8,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-16,-unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT16,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-4,+unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT4,ELT4-UNALIGNED,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-8,+unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT8,ELT8-UNALIGNED,ALL %s
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -mattr=+max-private-element-size-16,+unaligned-scratch-access -passes=load-store-vectorizer -S -o - %s | FileCheck -check-prefixes=ELT16,ELT16-UNALIGNED,ALL %s
+
+
+; NOTE: ELT8 runs store <2 x b32> twice for some functions.
+; NOTE: ELT16-UNALIGNED runs store <4 x b32> for some functions.
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32(ptr addrspace(5) %out) #0 {
+; ELT4-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32(
+; ELT4-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; ELT4-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 1
+; ELT4-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 2
+; ELT4-NEXT: [[OUT_GEP_3:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 3
+; ELT4-NEXT: store i32 9, ptr addrspace(5) [[OUT]], align 4
+; ELT4-NEXT: store i32 1, ptr addrspace(5) [[OUT_GEP_1]], align 4
+; ELT4-NEXT: store i32 23, ptr addrspace(5) [[OUT_GEP_2]], align 4
+; ELT4-NEXT: store i32 19, ptr addrspace(5) [[OUT_GEP_3]], align 4
+; ELT4-NEXT: ret void
+;
+; ELT8-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32(
+; ELT8-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; ELT8-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 2
+; ELT8-NEXT: store <2 x b32> <b32 9, b32 1>, ptr addrspace(5) [[OUT]], align 4
+; ELT8-NEXT: store <2 x b32> <b32 23, b32 19>, ptr addrspace(5) [[OUT_GEP_2]], align 4
+; ELT8-NEXT: ret void
+;
+; ELT16-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32(
+; ELT16-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; ELT16-NEXT: store <4 x b32> <b32 9, b32 1, b32 23, b32 19>, ptr addrspace(5) [[OUT]], align 4
+; ELT16-NEXT: ret void
+;
%out.gep.1 = getelementptr i32, ptr addrspace(5) %out, i32 1
%out.gep.2 = getelementptr i32, ptr addrspace(5) %out, i32 2
%out.gep.3 = getelementptr i32, ptr addrspace(5) %out, i32 3
@@ -27,22 +44,31 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32(ptr add
ret void
}
-; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32_align1(
-; ALIGNED: store i32 9, ptr addrspace(5) %out, align 1
-; ALIGNED: store i32 1, ptr addrspace(5) %out.gep.1, align 1
-; ALIGNED: store i32 23, ptr addrspace(5) %out.gep.2, align 1
-; ALIGNED: store i32 19, ptr addrspace(5) %out.gep.3, align 1
-
-; ELT16-UNALIGNED: store <4 x i32> <i32 9, i32 1, i32 23, i32 19>, ptr addrspace(5) %out, align 1
-
-; ELT8-UNALIGNED: store <2 x i32> <i32 9, i32 1>, ptr addrspace(5) %out, align 1
-; ELT8-UNALIGNED: store <2 x i32> <i32 23, i32 19>, ptr addrspace(5) %out.gep.2, align 1
-; ELT4-UNALIGNED: store i32
-; ELT4-UNALIGNED: store i32
-; ELT4-UNALIGNED: store i32
-; ELT4-UNALIGNED: store i32
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align1(ptr addrspace(5) %out) #0 {
+; ELT4-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align1(
+; ELT4-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 1
+; ELT4-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 2
+; ELT4-NEXT: [[OUT_GEP_3:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 3
+; ELT4-NEXT: store i32 9, ptr addrspace(5) [[OUT]], align 1
+; ELT4-NEXT: store i32 1, ptr addrspace(5) [[OUT_GEP_1]], align 1
+; ELT4-NEXT: store i32 23, ptr addrspace(5) [[OUT_GEP_2]], align 1
+; ELT4-NEXT: store i32 19, ptr addrspace(5) [[OUT_GEP_3]], align 1
+; ELT4-NEXT: ret void
+;
+; ELT8-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align1(
+; ELT8-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT8-UNALIGNED-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 2
+; ELT8-UNALIGNED-NEXT: store <2 x b32> <b32 9, b32 1>, ptr addrspace(5) [[OUT]], align 1
+; ELT8-UNALIGNED-NEXT: store <2 x b32> <b32 23, b32 19>, ptr addrspace(5) [[OUT_GEP_2]], align 1
+; ELT8-UNALIGNED-NEXT: ret void
+;
+; ELT16-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align1(
+; ELT16-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT16-UNALIGNED-NEXT: store <4 x b32> <b32 9, b32 1, b32 23, b32 19>, ptr addrspace(5) [[OUT]], align 1
+; ELT16-UNALIGNED-NEXT: ret void
+;
%out.gep.1 = getelementptr i32, ptr addrspace(5) %out, i32 1
%out.gep.2 = getelementptr i32, ptr addrspace(5) %out, i32 2
%out.gep.3 = getelementptr i32, ptr addrspace(5) %out, i32 3
@@ -54,19 +80,30 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align1(
ret void
}
-; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i32_align2(
-; ALIGNED: store i32
-; ALIGNED: store i32
-; ALIGNED: store i32
-; ALIGNED: store i32
-; ELT4-UNALIGNED: store i32
-; ELT4-UNALIGNED: store i32
-; ELT4-UNALIGNED: store i32
-; ELT4-UNALIGNED: store i32
-; ELT8-UNALIGNED: store <2 x i32>
-; ELT8-UNALIGNED: store <2 x i32>
-; ELT16-UNALIGNED: store <4 x i32>
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align2(ptr addrspace(5) %out) #0 {
+; ELT4-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align2(
+; ELT4-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 1
+; ELT4-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 2
+; ELT4-NEXT: [[OUT_GEP_3:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 3
+; ELT4-NEXT: store i32 9, ptr addrspace(5) [[OUT]], align 2
+; ELT4-NEXT: store i32 1, ptr addrspace(5) [[OUT_GEP_1]], align 2
+; ELT4-NEXT: store i32 23, ptr addrspace(5) [[OUT_GEP_2]], align 2
+; ELT4-NEXT: store i32 19, ptr addrspace(5) [[OUT_GEP_3]], align 2
+; ELT4-NEXT: ret void
+;
+; ELT8-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align2(
+; ELT8-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT8-UNALIGNED-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 2
+; ELT8-UNALIGNED-NEXT: store <2 x b32> <b32 9, b32 1>, ptr addrspace(5) [[OUT]], align 2
+; ELT8-UNALIGNED-NEXT: store <2 x b32> <b32 23, b32 19>, ptr addrspace(5) [[OUT_GEP_2]], align 2
+; ELT8-UNALIGNED-NEXT: ret void
+;
+; ELT16-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align2(
+; ELT16-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT16-UNALIGNED-NEXT: store <4 x b32> <b32 9, b32 1, b32 23, b32 19>, ptr addrspace(5) [[OUT]], align 2
+; ELT16-UNALIGNED-NEXT: ret void
+;
%out.gep.1 = getelementptr i32, ptr addrspace(5) %out, i32 1
%out.gep.2 = getelementptr i32, ptr addrspace(5) %out, i32 2
%out.gep.3 = getelementptr i32, ptr addrspace(5) %out, i32 3
@@ -78,9 +115,22 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i32_align2(
ret void
}
-; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i8(
-; ALL: store <4 x i8>
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8(ptr addrspace(5) %out) #0 {
+; ELT4-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8(
+; ELT4-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-NEXT: store <4 x b8> <b8 9, b8 1, b8 23, b8 19>, ptr addrspace(5) [[OUT]], align 4
+; ELT4-NEXT: ret void
+;
+; ELT8-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8(
+; ELT8-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT8-NEXT: store <4 x b8> <b8 9, b8 1, b8 23, b8 19>, ptr addrspace(5) [[OUT]], align 4
+; ELT8-NEXT: ret void
+;
+; ELT16-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8(
+; ELT16-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT16-NEXT: store <4 x b8> <b8 9, b8 1, b8 23, b8 19>, ptr addrspace(5) [[OUT]], align 4
+; ELT16-NEXT: ret void
+;
%out.gep.1 = getelementptr i8, ptr addrspace(5) %out, i32 1
%out.gep.2 = getelementptr i8, ptr addrspace(5) %out, i32 2
%out.gep.3 = getelementptr i8, ptr addrspace(5) %out, i32 3
@@ -92,14 +142,34 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8(ptr addr
ret void
}
-; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v4i8_align1(
-; ALIGNED: store i8
-; ALIGNED: store i8
-; ALIGNED: store i8
-; ALIGNED: store i8
-; UNALIGNED: store <4 x i8> <i8 9, i8 1, i8 23, i8 19>, ptr addrspace(5) %out, align 1
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8_align1(ptr addrspace(5) %out) #0 {
+; ELT4-ALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8_align1(
+; ELT4-ALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-ALIGNED-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i8, ptr addrspace(5) [[OUT]], i32 1
+; ELT4-ALIGNED-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i8, ptr addrspace(5) [[OUT]], i32 2
+; ELT4-ALIGNED-NEXT: [[OUT_GEP_3:%.*]] = getelementptr i8, ptr addrspace(5) [[OUT]], i32 3
+; ELT4-ALIGNED-NEXT: store i8 9, ptr addrspace(5) [[OUT]], align 1
+; ELT4-ALIGNED-NEXT: store i8 1, ptr addrspace(5) [[OUT_GEP_1]], align 1
+; ELT4-ALIGNED-NEXT: store i8 23, ptr addrspace(5) [[OUT_GEP_2]], align 1
+; ELT4-ALIGNED-NEXT: store i8 19, ptr addrspace(5) [[OUT_GEP_3]], align 1
+; ELT4-ALIGNED-NEXT: ret void
+;
+; ELT4-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8_align1(
+; ELT4-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-UNALIGNED-NEXT: store <4 x b8> <b8 9, b8 1, b8 23, b8 19>, ptr addrspace(5) [[OUT]], align 1
+; ELT4-UNALIGNED-NEXT: ret void
+;
+; ELT8-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8_align1(
+; ELT8-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT8-UNALIGNED-NEXT: store <4 x b8> <b8 9, b8 1, b8 23, b8 19>, ptr addrspace(5) [[OUT]], align 1
+; ELT8-UNALIGNED-NEXT: ret void
+;
+; ELT16-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8_align1(
+; ELT16-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT16-UNALIGNED-NEXT: store <4 x b8> <b8 9, b8 1, b8 23, b8 19>, ptr addrspace(5) [[OUT]], align 1
+; ELT16-UNALIGNED-NEXT: ret void
+;
%out.gep.1 = getelementptr i8, ptr addrspace(5) %out, i32 1
%out.gep.2 = getelementptr i8, ptr addrspace(5) %out, i32 2
%out.gep.3 = getelementptr i8, ptr addrspace(5) %out, i32 3
@@ -111,9 +181,22 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v4i8_align1(p
ret void
}
-; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v2i16(
-; ALL: store <2 x i16>
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16(ptr addrspace(5) %out) #0 {
+; ELT4-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16(
+; ELT4-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 4
+; ELT4-NEXT: ret void
+;
+; ELT8-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16(
+; ELT8-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT8-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 4
+; ELT8-NEXT: ret void
+;
+; ELT16-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16(
+; ELT16-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT16-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 4
+; ELT16-NEXT: ret void
+;
%out.gep.1 = getelementptr i16, ptr addrspace(5) %out, i32 1
store i16 9, ptr addrspace(5) %out, align 4
@@ -121,11 +204,29 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16(ptr add
ret void
}
-; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v2i16_align2(
-; ALIGNED: store i16
-; ALIGNED: store i16
-; UNALIGNED: store <2 x i16>
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align2(ptr addrspace(5) %out) #0 {
+; ELT4-ALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align2(
+; ELT4-ALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-ALIGNED-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i16, ptr addrspace(5) [[OUT]], i32 1
+; ELT4-ALIGNED-NEXT: store i16 9, ptr addrspace(5) [[OUT]], align 2
+; ELT4-ALIGNED-NEXT: store i16 12, ptr addrspace(5) [[OUT_GEP_1]], align 2
+; ELT4-ALIGNED-NEXT: ret void
+;
+; ELT4-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align2(
+; ELT4-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-UNALIGNED-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 2
+; ELT4-UNALIGNED-NEXT: ret void
+;
+; ELT8-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align2(
+; ELT8-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT8-UNALIGNED-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 2
+; ELT8-UNALIGNED-NEXT: ret void
+;
+; ELT16-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align2(
+; ELT16-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT16-UNALIGNED-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 2
+; ELT16-UNALIGNED-NEXT: ret void
+;
%out.gep.1 = getelementptr i16, ptr addrspace(5) %out, i32 1
store i16 9, ptr addrspace(5) %out, align 2
@@ -133,12 +234,30 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align2(
ret void
}
-; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v2i16_align1(
-; ALIGNED: store i16
-; ALIGNED: store i16
-; UNALIGNED: store <2 x i16> <i16 9, i16 12>, ptr addrspace(5) %out, align 1
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align1(ptr addrspace(5) %out) #0 {
+; ELT4-ALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align1(
+; ELT4-ALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-ALIGNED-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i16, ptr addrspace(5) [[OUT]], i32 1
+; ELT4-ALIGNED-NEXT: store i16 9, ptr addrspace(5) [[OUT]], align 1
+; ELT4-ALIGNED-NEXT: store i16 12, ptr addrspace(5) [[OUT_GEP_1]], align 1
+; ELT4-ALIGNED-NEXT: ret void
+;
+; ELT4-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align1(
+; ELT4-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-UNALIGNED-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 1
+; ELT4-UNALIGNED-NEXT: ret void
+;
+; ELT8-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align1(
+; ELT8-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT8-UNALIGNED-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 1
+; ELT8-UNALIGNED-NEXT: ret void
+;
+; ELT16-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align1(
+; ELT16-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT16-UNALIGNED-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 1
+; ELT16-UNALIGNED-NEXT: ret void
+;
%out.gep.1 = getelementptr i16, ptr addrspace(5) %out, i32 1
store i16 9, ptr addrspace(5) %out, align 1
@@ -146,9 +265,22 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align1(
ret void
}
-; ALL-LABEL: @merge_private_store_4_vector_elts_loads_v2i16_align8(
-; ALL: store <2 x i16> <i16 9, i16 12>, ptr addrspace(5) %out, align 8
define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align8(ptr addrspace(5) %out) #0 {
+; ELT4-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align8(
+; ELT4-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 8
+; ELT4-NEXT: ret void
+;
+; ELT8-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align8(
+; ELT8-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT8-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 8
+; ELT8-NEXT: ret void
+;
+; ELT16-LABEL: define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align8(
+; ELT16-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT16-NEXT: store <2 x b16> <b16 9, b16 12>, ptr addrspace(5) [[OUT]], align 8
+; ELT16-NEXT: ret void
+;
%out.gep.1 = getelementptr i16, ptr addrspace(5) %out, i32 1
store i16 9, ptr addrspace(5) %out, align 8
@@ -156,16 +288,30 @@ define amdgpu_kernel void @merge_private_store_4_vector_elts_loads_v2i16_align8(
ret void
}
-; ALL-LABEL: @merge_private_store_3_vector_elts_loads_v4i32
-; ELT4: store i32
-; ELT4: store i32
-; ELT4: store i32
-; ELT8: store <2 x i32>
-; ELT8: store i32
-
-; ELT16: store <3 x i32>
+; NOTE: merge_private_store_3_vector_elts: ELT8 uses <2 x b32> + scalar i32; ELT16 uses <3 x b32>.
define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32(ptr addrspace(5) %out) #0 {
+; ELT4-LABEL: define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32(
+; ELT4-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 1
+; ELT4-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 2
+; ELT4-NEXT: store i32 9, ptr addrspace(5) [[OUT]], align 4
+; ELT4-NEXT: store i32 1, ptr addrspace(5) [[OUT_GEP_1]], align 4
+; ELT4-NEXT: store i32 23, ptr addrspace(5) [[OUT_GEP_2]], align 4
+; ELT4-NEXT: ret void
+;
+; ELT8-LABEL: define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32(
+; ELT8-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT8-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 2
+; ELT8-NEXT: store <2 x b32> <b32 9, b32 1>, ptr addrspace(5) [[OUT]], align 4
+; ELT8-NEXT: store i32 23, ptr addrspace(5) [[OUT_GEP_2]], align 4
+; ELT8-NEXT: ret void
+;
+; ELT16-LABEL: define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32(
+; ELT16-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT16-NEXT: store <3 x b32> <b32 9, b32 1, b32 23>, ptr addrspace(5) [[OUT]], align 4
+; ELT16-NEXT: ret void
+;
%out.gep.1 = getelementptr i32, ptr addrspace(5) %out, i32 1
%out.gep.2 = getelementptr i32, ptr addrspace(5) %out, i32 2
@@ -175,20 +321,29 @@ define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32(ptr add
ret void
}
-; ALL-LABEL: @merge_private_store_3_vector_elts_loads_v4i32_align1(
-; ALIGNED: store i32
-; ALIGNED: store i32
-; ALIGNED: store i32
-
-; ELT4-UNALIGNED: store i32
-; ELT4-UNALIGNED: store i32
-; ELT4-UNALIGNED: store i32
-
-; ELT8-UNALIGNED: store <2 x i32>
-; ELT8-UNALIGNED: store i32
-; ELT16-UNALIGNED: store <3 x i32>
define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32_align1(ptr addrspace(5) %out) #0 {
+; ELT4-LABEL: define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32_align1(
+; ELT4-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 1
+; ELT4-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 2
+; ELT4-NEXT: store i32 9, ptr addrspace(5) [[OUT]], align 1
+; ELT4-NEXT: store i32 1, ptr addrspace(5) [[OUT_GEP_1]], align 1
+; ELT4-NEXT: store i32 23, ptr addrspace(5) [[OUT_GEP_2]], align 1
+; ELT4-NEXT: ret void
+;
+; ELT8-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32_align1(
+; ELT8-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT8-UNALIGNED-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(5) [[OUT]], i32 2
+; ELT8-UNALIGNED-NEXT: store <2 x b32> <b32 9, b32 1>, ptr addrspace(5) [[OUT]], align 1
+; ELT8-UNALIGNED-NEXT: store i32 23, ptr addrspace(5) [[OUT_GEP_2]], align 1
+; ELT8-UNALIGNED-NEXT: ret void
+;
+; ELT16-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32_align1(
+; ELT16-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT16-UNALIGNED-NEXT: store <3 x b32> <b32 9, b32 1, b32 23>, ptr addrspace(5) [[OUT]], align 1
+; ELT16-UNALIGNED-NEXT: ret void
+;
%out.gep.1 = getelementptr i32, ptr addrspace(5) %out, i32 1
%out.gep.2 = getelementptr i32, ptr addrspace(5) %out, i32 2
@@ -198,13 +353,32 @@ define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i32_align1(
ret void
}
-; ALL-LABEL: @merge_private_store_3_vector_elts_loads_v4i8_align1(
-; ALIGNED: store i8
-; ALIGNED: store i8
-; ALIGNED: store i8
-; UNALIGNED: store <3 x i8>
define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i8_align1(ptr addrspace(5) %out) #0 {
+; ELT4-ALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i8_align1(
+; ELT4-ALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-ALIGNED-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i8, ptr addrspace(5) [[OUT]], i8 1
+; ELT4-ALIGNED-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i8, ptr addrspace(5) [[OUT]], i8 2
+; ELT4-ALIGNED-NEXT: store i8 9, ptr addrspace(5) [[OUT]], align 1
+; ELT4-ALIGNED-NEXT: store i8 1, ptr addrspace(5) [[OUT_GEP_1]], align 1
+; ELT4-ALIGNED-NEXT: store i8 23, ptr addrspace(5) [[OUT_GEP_2]], align 1
+; ELT4-ALIGNED-NEXT: ret void
+;
+; ELT4-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i8_align1(
+; ELT4-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT4-UNALIGNED-NEXT: store <3 x b8> <b8 9, b8 1, b8 23>, ptr addrspace(5) [[OUT]], align 1
+; ELT4-UNALIGNED-NEXT: ret void
+;
+; ELT8-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i8_align1(
+; ELT8-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT8-UNALIGNED-NEXT: store <3 x b8> <b8 9, b8 1, b8 23>, ptr addrspace(5) [[OUT]], align 1
+; ELT8-UNALIGNED-NEXT: ret void
+;
+; ELT16-UNALIGNED-LABEL: define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i8_align1(
+; ELT16-UNALIGNED-SAME: ptr addrspace(5) [[OUT:%.*]]) #[[ATTR0]] {
+; ELT16-UNALIGNED-NEXT: store <3 x b8> <b8 9, b8 1, b8 23>, ptr addrspace(5) [[OUT]], align 1
+; ELT16-UNALIGNED-NEXT: ret void
+;
%out.gep.1 = getelementptr i8, ptr addrspace(5) %out, i8 1
%out.gep.2 = getelementptr i8, ptr addrspace(5) %out, i8 2
@@ -215,3 +389,5 @@ define amdgpu_kernel void @merge_private_store_3_vector_elts_loads_v4i8_align1(p
}
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; ALL: {{.*}}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
index 6c9f888d69849..4184aaaf69a19 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-stores.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa --mcpu=hawaii -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; Copy of test/CodeGen/AMDGPU/merge-stores.ll with some additions
@@ -7,8 +8,9 @@
define amdgpu_kernel void @merge_global_store_2_constants_i8(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_i8(
-; CHECK-NEXT: store <2 x i8> <i8 -56, i8 123>, ptr addrspace(1) [[OUT:%.*]], align 2
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_i8(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: store <2 x b8> <b8 -56, b8 123>, ptr addrspace(1) [[OUT]], align 2
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i8, ptr addrspace(1) %out, i32 1
@@ -19,8 +21,9 @@ define amdgpu_kernel void @merge_global_store_2_constants_i8(ptr addrspace(1) %o
}
define amdgpu_kernel void @merge_global_store_2_constants_i8_natural_align(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_i8_natural_align(
-; CHECK-NEXT: store <2 x i8> <i8 -56, i8 123>, ptr addrspace(1) [[OUT:%.*]], align 1
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_i8_natural_align(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b8> <b8 -56, b8 123>, ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i8, ptr addrspace(1) %out, i32 1
@@ -31,8 +34,9 @@ define amdgpu_kernel void @merge_global_store_2_constants_i8_natural_align(ptr a
}
define amdgpu_kernel void @merge_global_store_2_constants_i16(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_i16(
-; CHECK-NEXT: store <2 x i16> <i16 456, i16 123>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_i16(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b16> <b16 456, b16 123>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i16, ptr addrspace(1) %out, i32 1
@@ -43,8 +47,9 @@ define amdgpu_kernel void @merge_global_store_2_constants_i16(ptr addrspace(1) %
}
define amdgpu_kernel void @merge_global_store_2_constants_0_i16(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_0_i16(
-; CHECK-NEXT: store <2 x i16> zeroinitializer, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_0_i16(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b16> zeroinitializer, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i16, ptr addrspace(1) %out, i32 1
@@ -55,8 +60,9 @@ define amdgpu_kernel void @merge_global_store_2_constants_0_i16(ptr addrspace(1)
}
define amdgpu_kernel void @merge_global_store_2_constants_i16_natural_align(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_i16_natural_align(
-; CHECK-NEXT: store <2 x i16> <i16 456, i16 123>, ptr addrspace(1) [[OUT:%.*]], align 2
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_i16_natural_align(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b16> <b16 456, b16 123>, ptr addrspace(1) [[OUT]], align 2
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i16, ptr addrspace(1) %out, i32 1
@@ -67,8 +73,9 @@ define amdgpu_kernel void @merge_global_store_2_constants_i16_natural_align(ptr
}
define amdgpu_kernel void @merge_global_store_2_constants_i16_align_1(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_i16_align_1(
-; CHECK-NEXT: store <2 x i16> <i16 456, i16 123>, ptr addrspace(1) [[OUT:%.*]], align 1
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_i16_align_1(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b16> <b16 456, b16 123>, ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i16, ptr addrspace(1) %out, i32 1
@@ -79,8 +86,9 @@ define amdgpu_kernel void @merge_global_store_2_constants_i16_align_1(ptr addrsp
}
define amdgpu_kernel void @merge_global_store_2_constants_half_natural_align(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_half_natural_align(
-; CHECK-NEXT: store <2 x half> <half 1.000000e+00, half 2.000000e+00>, ptr addrspace(1) [[OUT:%.*]], align 2
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_half_natural_align(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b16> <b16 15360, b16 16384>, ptr addrspace(1) [[OUT]], align 2
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr half, ptr addrspace(1) %out, i32 1
@@ -91,8 +99,9 @@ define amdgpu_kernel void @merge_global_store_2_constants_half_natural_align(ptr
}
define amdgpu_kernel void @merge_global_store_2_constants_half_align_1(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_half_align_1(
-; CHECK-NEXT: store <2 x half> <half 1.000000e+00, half 2.000000e+00>, ptr addrspace(1) [[OUT:%.*]], align 1
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_half_align_1(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b16> <b16 15360, b16 16384>, ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr half, ptr addrspace(1) %out, i32 1
@@ -103,8 +112,9 @@ define amdgpu_kernel void @merge_global_store_2_constants_half_align_1(ptr addrs
}
define amdgpu_kernel void @merge_global_store_2_constants_i32(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_i32(
-; CHECK-NEXT: store <2 x i32> <i32 456, i32 123>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b32> <b32 456, b32 123>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
@@ -115,8 +125,9 @@ define amdgpu_kernel void @merge_global_store_2_constants_i32(ptr addrspace(1) %
}
define amdgpu_kernel void @merge_global_store_2_constants_i32_f32(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_i32_f32(
-; CHECK-NEXT: store <2 x i32> <i32 456, i32 1065353216>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_i32_f32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b32> <b32 456, b32 1065353216>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
@@ -126,8 +137,9 @@ define amdgpu_kernel void @merge_global_store_2_constants_i32_f32(ptr addrspace(
}
define amdgpu_kernel void @merge_global_store_2_constants_f32_i32(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_f32_i32(
-; CHECK-NEXT: store <2 x i32> <i32 1082130432, i32 123>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_f32_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b32> <b32 1082130432, b32 123>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr float, ptr addrspace(1) %out, i32 1
@@ -137,8 +149,9 @@ define amdgpu_kernel void @merge_global_store_2_constants_f32_i32(ptr addrspace(
}
define amdgpu_kernel void @merge_global_store_4_constants_i32(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_4_constants_i32(
-; CHECK-NEXT: store <4 x i32> <i32 1234, i32 123, i32 456, i32 333>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_constants_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <4 x b32> <b32 1234, b32 123, b32 456, b32 333>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
@@ -153,8 +166,9 @@ define amdgpu_kernel void @merge_global_store_4_constants_i32(ptr addrspace(1) %
}
define amdgpu_kernel void @merge_global_store_4_constants_f32_order(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_4_constants_f32_order(
-; CHECK-NEXT: store <4 x float> <float 8.000000e+00, float 1.000000e+00, float 2.000000e+00, float 4.000000e+00>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_constants_f32_order(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <4 x b32> <b32 1090519040, b32 1065353216, b32 1073741824, b32 1082130432>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr float, ptr addrspace(1) %out, i32 1
@@ -170,8 +184,9 @@ define amdgpu_kernel void @merge_global_store_4_constants_f32_order(ptr addrspac
; First store is out of order.
define amdgpu_kernel void @merge_global_store_4_constants_f32(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_4_constants_f32(
-; CHECK-NEXT: store <4 x float> <float 8.000000e+00, float 1.000000e+00, float 2.000000e+00, float 4.000000e+00>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_constants_f32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <4 x b32> <b32 1090519040, b32 1065353216, b32 1073741824, b32 1082130432>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr float, ptr addrspace(1) %out, i32 1
@@ -186,8 +201,9 @@ define amdgpu_kernel void @merge_global_store_4_constants_f32(ptr addrspace(1) %
}
define amdgpu_kernel void @merge_global_store_4_constants_mixed_i32_f32(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_4_constants_mixed_i32_f32(
-; CHECK-NEXT: store <4 x i32> <i32 1090519040, i32 11, i32 1073741824, i32 17>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_constants_mixed_i32_f32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <4 x b32> <b32 1090519040, b32 11, b32 1073741824, b32 17>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr float, ptr addrspace(1) %out, i32 1
@@ -203,8 +219,9 @@ define amdgpu_kernel void @merge_global_store_4_constants_mixed_i32_f32(ptr addr
}
define amdgpu_kernel void @merge_global_store_3_constants_i32(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_3_constants_i32(
-; CHECK-NEXT: store <3 x i32> <i32 1234, i32 123, i32 456>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_3_constants_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <3 x b32> <b32 1234, b32 123, b32 456>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
@@ -217,8 +234,9 @@ define amdgpu_kernel void @merge_global_store_3_constants_i32(ptr addrspace(1) %
}
define amdgpu_kernel void @merge_global_store_2_constants_i64(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_2_constants_i64(
-; CHECK-NEXT: store <2 x i64> <i64 456, i64 123>, ptr addrspace(1) [[OUT:%.*]], align 8
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_constants_i64(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b64> <b64 456, b64 123>, ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i64, ptr addrspace(1) %out, i64 1
@@ -229,10 +247,11 @@ define amdgpu_kernel void @merge_global_store_2_constants_i64(ptr addrspace(1) %
}
define amdgpu_kernel void @merge_global_store_4_constants_i64(ptr addrspace(1) %out) #0 {
-; CHECK-LABEL: @merge_global_store_4_constants_i64(
-; CHECK-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i64, ptr addrspace(1) [[OUT:%.*]], i64 2
-; CHECK-NEXT: store <2 x i64> <i64 456, i64 333>, ptr addrspace(1) [[OUT_GEP_2]], align 8
-; CHECK-NEXT: store <2 x i64> <i64 1234, i64 123>, ptr addrspace(1) [[OUT]], align 8
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_constants_i64(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i64, ptr addrspace(1) [[OUT]], i64 2
+; CHECK-NEXT: store <2 x b64> <b64 456, b64 333>, ptr addrspace(1) [[OUT_GEP_2]], align 8
+; CHECK-NEXT: store <2 x b64> <b64 1234, b64 123>, ptr addrspace(1) [[OUT]], align 8
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i64, ptr addrspace(1) %out, i64 1
@@ -247,13 +266,18 @@ define amdgpu_kernel void @merge_global_store_4_constants_i64(ptr addrspace(1) %
}
define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_2_adjacent_loads_i32(
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[IN:%.*]], align 4
-; CHECK-NEXT: [[LO1:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[HI2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[LO1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[HI2]], i32 1
-; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(1) [[IN]], align 4
+; CHECK-NEXT: [[LO1:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LO1]] to i32
+; CHECK-NEXT: [[HI2:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[HI2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x b32> poison, b32 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[TMP3]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x b32> [[TMP5]], b32 [[TMP6]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP7]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
@@ -268,15 +292,20 @@ define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32(ptr addrspace
}
define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32_nonzero_base(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_2_adjacent_loads_i32_nonzero_base(
-; CHECK-NEXT: [[IN_GEP_0:%.*]] = getelementptr i32, ptr addrspace(1) [[IN:%.*]], i32 2
-; CHECK-NEXT: [[OUT_GEP_0:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i32 2
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[IN_GEP_0]], align 4
-; CHECK-NEXT: [[LO1:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[HI2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[LO1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[HI2]], i32 1
-; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr addrspace(1) [[OUT_GEP_0]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32_nonzero_base(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[IN_GEP_0:%.*]] = getelementptr i32, ptr addrspace(1) [[IN]], i32 2
+; CHECK-NEXT: [[OUT_GEP_0:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 2
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(1) [[IN_GEP_0]], align 4
+; CHECK-NEXT: [[LO1:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LO1]] to i32
+; CHECK-NEXT: [[HI2:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[HI2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x b32> poison, b32 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[TMP3]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x b32> [[TMP5]], b32 [[TMP6]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP7]], ptr addrspace(1) [[OUT_GEP_0]], align 4
; CHECK-NEXT: ret void
;
%in.gep.0 = getelementptr i32, ptr addrspace(1) %in, i32 2
@@ -293,13 +322,18 @@ define amdgpu_kernel void @merge_global_store_2_adjacent_loads_i32_nonzero_base(
}
define amdgpu_kernel void @merge_global_store_2_adjacent_loads_shuffle_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_2_adjacent_loads_shuffle_i32(
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[IN:%.*]], align 4
-; CHECK-NEXT: [[LO1:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[HI2:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[HI2]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[LO1]], i32 1
-; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_2_adjacent_loads_shuffle_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(1) [[IN]], align 4
+; CHECK-NEXT: [[LO1:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LO1]] to i32
+; CHECK-NEXT: [[HI2:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[HI2]] to i32
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32 [[TMP3]] to b32
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x b32> poison, b32 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x b32> [[TMP5]], b32 [[TMP6]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP7]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
@@ -314,17 +348,26 @@ define amdgpu_kernel void @merge_global_store_2_adjacent_loads_shuffle_i32(ptr a
}
define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_4_adjacent_loads_i32(
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr addrspace(1) [[IN:%.*]], align 4
-; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[Y2]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[Z3]], i32 2
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[W4]], i32 3
-; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(1) [[IN]], align 4
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[X1]] to i32
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[Y2]] to i32
+; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[Z3]] to i32
+; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[W4]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x b32> poison, b32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP3]] to b32
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x b32> [[TMP7]], b32 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP4]] to b32
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x b32> [[TMP9]], b32 [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32 [[TMP5]] to b32
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x b32> [[TMP11]], b32 [[TMP12]], i32 3
+; CHECK-NEXT: store <4 x b32> [[TMP13]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
@@ -347,15 +390,22 @@ define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i32(ptr addrspace
}
define amdgpu_kernel void @merge_global_store_3_adjacent_loads_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_3_adjacent_loads_i32(
-; CHECK-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr addrspace(1) [[IN:%.*]], align 4
-; CHECK-NEXT: [[X1:%.*]] = extractelement <3 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[Y2:%.*]] = extractelement <3 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[Z3:%.*]] = extractelement <3 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <3 x i32> poison, i32 [[X1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <3 x i32> [[TMP2]], i32 [[Y2]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <3 x i32> [[TMP3]], i32 [[Z3]], i32 2
-; CHECK-NEXT: store <3 x i32> [[TMP4]], ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_3_adjacent_loads_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <3 x b32>, ptr addrspace(1) [[IN]], align 4
+; CHECK-NEXT: [[X1:%.*]] = extractelement <3 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[X1]] to i32
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <3 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[Y2]] to i32
+; CHECK-NEXT: [[Z3:%.*]] = extractelement <3 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[Z3]] to i32
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <3 x b32> poison, b32 [[TMP5]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32 [[TMP3]] to b32
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <3 x b32> [[TMP6]], b32 [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast i32 [[TMP4]] to b32
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <3 x b32> [[TMP8]], b32 [[TMP9]], i32 2
+; CHECK-NEXT: store <3 x b32> [[TMP10]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
@@ -374,17 +424,26 @@ define amdgpu_kernel void @merge_global_store_3_adjacent_loads_i32(ptr addrspace
}
define amdgpu_kernel void @merge_global_store_4_adjacent_loads_f32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_4_adjacent_loads_f32(
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[IN:%.*]], align 4
-; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
-; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
-; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
-; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x float> poison, float [[X1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> [[TMP2]], float [[Y2]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[Z3]], i32 2
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[W4]], i32 3
-; CHECK-NEXT: store <4 x float> [[TMP5]], ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_adjacent_loads_f32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(1) [[IN]], align 4
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[X1]] to float
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[Y2]] to float
+; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[Z3]] to float
+; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[W4]] to float
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast float [[TMP2]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x b32> poison, b32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast float [[TMP3]] to b32
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x b32> [[TMP7]], b32 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast float [[TMP4]] to b32
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x b32> [[TMP9]], b32 [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast float [[TMP5]] to b32
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x b32> [[TMP11]], b32 [[TMP12]], i32 3
+; CHECK-NEXT: store <4 x b32> [[TMP13]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr float, ptr addrspace(1) %out, i32 1
@@ -407,19 +466,28 @@ define amdgpu_kernel void @merge_global_store_4_adjacent_loads_f32(ptr addrspace
}
define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i32_nonzero_base(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_4_adjacent_loads_i32_nonzero_base(
-; CHECK-NEXT: [[IN_GEP_0:%.*]] = getelementptr i32, ptr addrspace(1) [[IN:%.*]], i32 11
-; CHECK-NEXT: [[OUT_GEP_0:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT:%.*]], i32 7
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr addrspace(1) [[IN_GEP_0]], align 4
-; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[Y2]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[Z3]], i32 2
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[W4]], i32 3
-; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr addrspace(1) [[OUT_GEP_0]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i32_nonzero_base(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[IN_GEP_0:%.*]] = getelementptr i32, ptr addrspace(1) [[IN]], i32 11
+; CHECK-NEXT: [[OUT_GEP_0:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 7
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(1) [[IN_GEP_0]], align 4
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[X1]] to i32
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[Y2]] to i32
+; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[Z3]] to i32
+; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[W4]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x b32> poison, b32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP3]] to b32
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x b32> [[TMP7]], b32 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP4]] to b32
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x b32> [[TMP9]], b32 [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32 [[TMP5]] to b32
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x b32> [[TMP11]], b32 [[TMP12]], i32 3
+; CHECK-NEXT: store <4 x b32> [[TMP13]], ptr addrspace(1) [[OUT_GEP_0]], align 4
; CHECK-NEXT: ret void
;
%in.gep.0 = getelementptr i32, ptr addrspace(1) %in, i32 11
@@ -444,18 +512,27 @@ define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i32_nonzero_base(
}
define amdgpu_kernel void @merge_global_store_4_adjacent_loads_inverse_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_4_adjacent_loads_inverse_i32(
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr addrspace(1) [[IN:%.*]], align 4
-; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_adjacent_loads_inverse_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(1) [[IN]], align 4
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[X1]] to i32
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[Y2]] to i32
+; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[Z3]] to i32
+; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[W4]] to i32
; CHECK-NEXT: tail call void @llvm.amdgcn.s.barrier() #[[ATTR3:[0-9]+]]
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[X1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[Y2]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[Z3]], i32 2
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[W4]], i32 3
-; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x b32> poison, b32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP3]] to b32
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x b32> [[TMP7]], b32 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP4]] to b32
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x b32> [[TMP9]], b32 [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32 [[TMP5]] to b32
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x b32> [[TMP11]], b32 [[TMP12]], i32 3
+; CHECK-NEXT: store <4 x b32> [[TMP13]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
@@ -482,18 +559,27 @@ define amdgpu_kernel void @merge_global_store_4_adjacent_loads_inverse_i32(ptr a
}
define amdgpu_kernel void @merge_global_store_4_adjacent_loads_shuffle_i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_4_adjacent_loads_shuffle_i32(
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr addrspace(1) [[IN:%.*]], align 4
-; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_adjacent_loads_shuffle_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(1) [[IN]], align 4
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[X1]] to i32
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[Y2]] to i32
+; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[Z3]] to i32
+; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[W4]] to i32
; CHECK-NEXT: tail call void @llvm.amdgcn.s.barrier() #[[ATTR3]]
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> poison, i32 [[W4]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[Z3]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[Y2]], i32 2
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[X1]], i32 3
-; CHECK-NEXT: store <4 x i32> [[TMP5]], ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[TMP5]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x b32> poison, b32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[TMP4]] to b32
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x b32> [[TMP7]], b32 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP3]] to b32
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x b32> [[TMP9]], b32 [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x b32> [[TMP11]], b32 [[TMP12]], i32 3
+; CHECK-NEXT: store <4 x b32> [[TMP13]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
@@ -520,17 +606,26 @@ define amdgpu_kernel void @merge_global_store_4_adjacent_loads_shuffle_i32(ptr a
}
define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i8(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_4_adjacent_loads_i8(
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN:%.*]], align 4
-; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i8> poison, i8 [[X1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i8> [[TMP2]], i8 [[Y2]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i8> [[TMP3]], i8 [[Z3]], i32 2
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i8> [[TMP4]], i8 [[W4]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP5]], ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i8(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b8>, ptr addrspace(1) [[IN]], align 4
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x b8> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[X1]] to i8
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x b8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[Y2]] to i8
+; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x b8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b8 [[Z3]] to i8
+; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x b8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b8 [[W4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i8 [[TMP2]] to b8
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x b8> poison, b8 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i8 [[TMP3]] to b8
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x b8> [[TMP7]], b8 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i8 [[TMP4]] to b8
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x b8> [[TMP9]], b8 [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast i8 [[TMP5]] to b8
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x b8> [[TMP11]], b8 [[TMP12]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP13]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i8, ptr addrspace(1) %out, i8 1
@@ -553,17 +648,26 @@ define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i8(ptr addrspace(
}
define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i8_natural_align(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_4_adjacent_loads_i8_natural_align(
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr addrspace(1) [[IN:%.*]], align 1
-; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i8> poison, i8 [[X1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i8> [[TMP2]], i8 [[Y2]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i8> [[TMP3]], i8 [[Z3]], i32 2
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i8> [[TMP4]], i8 [[W4]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP5]], ptr addrspace(1) [[OUT:%.*]], align 1
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i8_natural_align(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b8>, ptr addrspace(1) [[IN]], align 1
+; CHECK-NEXT: [[X1:%.*]] = extractelement <4 x b8> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[X1]] to i8
+; CHECK-NEXT: [[Y2:%.*]] = extractelement <4 x b8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[Y2]] to i8
+; CHECK-NEXT: [[Z3:%.*]] = extractelement <4 x b8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b8 [[Z3]] to i8
+; CHECK-NEXT: [[W4:%.*]] = extractelement <4 x b8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b8 [[W4]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i8 [[TMP2]] to b8
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x b8> poison, b8 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i8 [[TMP3]] to b8
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x b8> [[TMP7]], b8 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i8 [[TMP4]] to b8
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x b8> [[TMP9]], b8 [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast i8 [[TMP5]] to b8
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x b8> [[TMP11]], b8 [[TMP12]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP13]], ptr addrspace(1) [[OUT]], align 1
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i8, ptr addrspace(1) %out, i8 1
@@ -586,17 +690,22 @@ define amdgpu_kernel void @merge_global_store_4_adjacent_loads_i8_natural_align(
}
define amdgpu_kernel void @merge_global_store_4_vector_elts_loads_v4i32(ptr addrspace(1) %out, ptr addrspace(1) %in) #0 {
-; CHECK-LABEL: @merge_global_store_4_vector_elts_loads_v4i32(
-; CHECK-NEXT: [[VEC:%.*]] = load <4 x i32>, ptr addrspace(1) [[IN:%.*]], align 16
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_4_vector_elts_loads_v4i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[VEC:%.*]] = load <4 x i32>, ptr addrspace(1) [[IN]], align 16
; CHECK-NEXT: [[X:%.*]] = extractelement <4 x i32> [[VEC]], i32 0
; CHECK-NEXT: [[Y:%.*]] = extractelement <4 x i32> [[VEC]], i32 1
; CHECK-NEXT: [[Z:%.*]] = extractelement <4 x i32> [[VEC]], i32 2
; CHECK-NEXT: [[W:%.*]] = extractelement <4 x i32> [[VEC]], i32 3
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x i32> poison, i32 [[X]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x i32> [[TMP1]], i32 [[Y]], i32 1
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> [[TMP2]], i32 [[Z]], i32 2
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[W]], i32 3
-; CHECK-NEXT: store <4 x i32> [[TMP4]], ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast i32 [[X]] to b32
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x b32> poison, b32 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32 [[Y]] to b32
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x b32> [[TMP2]], b32 [[TMP3]], i32 1
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i32 [[Z]] to b32
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x b32> [[TMP4]], b32 [[TMP5]], i32 2
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32 [[W]] to b32
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x b32> [[TMP6]], b32 [[TMP7]], i32 3
+; CHECK-NEXT: store <4 x b32> [[TMP8]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
@@ -617,8 +726,9 @@ define amdgpu_kernel void @merge_global_store_4_vector_elts_loads_v4i32(ptr addr
}
define amdgpu_kernel void @merge_local_store_2_constants_i8(ptr addrspace(3) %out) #0 {
-; CHECK-LABEL: @merge_local_store_2_constants_i8(
-; CHECK-NEXT: store <2 x i8> <i8 -56, i8 123>, ptr addrspace(3) [[OUT:%.*]], align 2
+; CHECK-LABEL: define amdgpu_kernel void @merge_local_store_2_constants_i8(
+; CHECK-SAME: ptr addrspace(3) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b8> <b8 -56, b8 123>, ptr addrspace(3) [[OUT]], align 2
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i8, ptr addrspace(3) %out, i32 1
@@ -629,8 +739,9 @@ define amdgpu_kernel void @merge_local_store_2_constants_i8(ptr addrspace(3) %ou
}
define amdgpu_kernel void @merge_local_store_2_constants_i32(ptr addrspace(3) %out) #0 {
-; CHECK-LABEL: @merge_local_store_2_constants_i32(
-; CHECK-NEXT: store <2 x i32> <i32 456, i32 123>, ptr addrspace(3) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_local_store_2_constants_i32(
+; CHECK-SAME: ptr addrspace(3) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: store <2 x b32> <b32 456, b32 123>, ptr addrspace(3) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(3) %out, i32 1
@@ -641,8 +752,9 @@ define amdgpu_kernel void @merge_local_store_2_constants_i32(ptr addrspace(3) %o
}
define amdgpu_kernel void @merge_local_store_2_constants_i32_align_2(ptr addrspace(3) %out) #0 {
-; CHECK-LABEL: @merge_local_store_2_constants_i32_align_2(
-; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i32, ptr addrspace(3) [[OUT:%.*]], i32 1
+; CHECK-LABEL: define amdgpu_kernel void @merge_local_store_2_constants_i32_align_2(
+; CHECK-SAME: ptr addrspace(3) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i32, ptr addrspace(3) [[OUT]], i32 1
; CHECK-NEXT: store i32 123, ptr addrspace(3) [[OUT_GEP_1]], align 2
; CHECK-NEXT: store i32 456, ptr addrspace(3) [[OUT]], align 2
; CHECK-NEXT: ret void
@@ -655,10 +767,11 @@ define amdgpu_kernel void @merge_local_store_2_constants_i32_align_2(ptr addrspa
}
define amdgpu_kernel void @merge_local_store_4_constants_i32(ptr addrspace(3) %out) #0 {
-; CHECK-LABEL: @merge_local_store_4_constants_i32(
-; CHECK-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(3) [[OUT:%.*]], i32 2
-; CHECK-NEXT: store <2 x i32> <i32 456, i32 333>, ptr addrspace(3) [[OUT_GEP_2]], align 4
-; CHECK-NEXT: store <2 x i32> <i32 1234, i32 123>, ptr addrspace(3) [[OUT]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_local_store_4_constants_i32(
+; CHECK-SAME: ptr addrspace(3) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(3) [[OUT]], i32 2
+; CHECK-NEXT: store <2 x b32> <b32 456, b32 333>, ptr addrspace(3) [[OUT_GEP_2]], align 4
+; CHECK-NEXT: store <2 x b32> <b32 1234, b32 123>, ptr addrspace(3) [[OUT]], align 4
; CHECK-NEXT: ret void
;
%out.gep.1 = getelementptr i32, ptr addrspace(3) %out, i32 1
@@ -673,8 +786,9 @@ define amdgpu_kernel void @merge_local_store_4_constants_i32(ptr addrspace(3) %o
}
define amdgpu_kernel void @merge_global_store_5_constants_i32(ptr addrspace(1) %out) {
-; CHECK-LABEL: @merge_global_store_5_constants_i32(
-; CHECK-NEXT: store <4 x i32> <i32 9, i32 12, i32 16, i32 -12>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_5_constants_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: store <4 x b32> <b32 9, b32 12, b32 16, b32 -12>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: [[IDX4:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 4
; CHECK-NEXT: store i32 11, ptr addrspace(1) [[IDX4]], align 4
; CHECK-NEXT: ret void
@@ -692,10 +806,11 @@ define amdgpu_kernel void @merge_global_store_5_constants_i32(ptr addrspace(1) %
}
define amdgpu_kernel void @merge_global_store_6_constants_i32(ptr addrspace(1) %out) {
-; CHECK-LABEL: @merge_global_store_6_constants_i32(
-; CHECK-NEXT: store <4 x i32> <i32 13, i32 15, i32 62, i32 63>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_6_constants_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: store <4 x b32> <b32 13, b32 15, b32 62, b32 63>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: [[IDX4:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 4
-; CHECK-NEXT: store <2 x i32> <i32 11, i32 123>, ptr addrspace(1) [[IDX4]], align 4
+; CHECK-NEXT: store <2 x b32> <b32 11, b32 123>, ptr addrspace(1) [[IDX4]], align 4
; CHECK-NEXT: ret void
;
store i32 13, ptr addrspace(1) %out, align 4
@@ -713,10 +828,11 @@ define amdgpu_kernel void @merge_global_store_6_constants_i32(ptr addrspace(1) %
}
define amdgpu_kernel void @merge_global_store_7_constants_i32(ptr addrspace(1) %out) {
-; CHECK-LABEL: @merge_global_store_7_constants_i32(
-; CHECK-NEXT: store <4 x i32> <i32 34, i32 999, i32 65, i32 33>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_7_constants_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: store <4 x b32> <b32 34, b32 999, b32 65, b32 33>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: [[IDX4:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 4
-; CHECK-NEXT: store <3 x i32> <i32 98, i32 91, i32 212>, ptr addrspace(1) [[IDX4]], align 4
+; CHECK-NEXT: store <3 x b32> <b32 98, b32 91, b32 212>, ptr addrspace(1) [[IDX4]], align 4
; CHECK-NEXT: ret void
;
store i32 34, ptr addrspace(1) %out, align 4
@@ -736,10 +852,11 @@ define amdgpu_kernel void @merge_global_store_7_constants_i32(ptr addrspace(1) %
}
define amdgpu_kernel void @merge_global_store_8_constants_i32(ptr addrspace(1) %out) {
-; CHECK-LABEL: @merge_global_store_8_constants_i32(
-; CHECK-NEXT: store <4 x i32> <i32 34, i32 999, i32 65, i32 33>, ptr addrspace(1) [[OUT:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @merge_global_store_8_constants_i32(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: store <4 x b32> <b32 34, b32 999, b32 65, b32 33>, ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: [[IDX4:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[OUT]], i64 4
-; CHECK-NEXT: store <4 x i32> <i32 98, i32 91, i32 212, i32 999>, ptr addrspace(1) [[IDX4]], align 4
+; CHECK-NEXT: store <4 x b32> <b32 98, b32 91, b32 212, b32 999>, ptr addrspace(1) [[IDX4]], align 4
; CHECK-NEXT: ret void
;
store i32 34, ptr addrspace(1) %out, align 4
@@ -761,9 +878,10 @@ define amdgpu_kernel void @merge_global_store_8_constants_i32(ptr addrspace(1) %
}
define amdgpu_kernel void @copy_v3i32_align4(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #0 {
-; CHECK-LABEL: @copy_v3i32_align4(
-; CHECK-NEXT: [[VEC:%.*]] = load <3 x i32>, ptr addrspace(1) [[IN:%.*]], align 4
-; CHECK-NEXT: store <3 x i32> [[VEC]], ptr addrspace(1) [[OUT:%.*]], align 16
+; CHECK-LABEL: define amdgpu_kernel void @copy_v3i32_align4(
+; CHECK-SAME: ptr addrspace(1) noalias [[OUT:%.*]], ptr addrspace(1) noalias [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[VEC:%.*]] = load <3 x i32>, ptr addrspace(1) [[IN]], align 4
+; CHECK-NEXT: store <3 x i32> [[VEC]], ptr addrspace(1) [[OUT]], align 16
; CHECK-NEXT: ret void
;
%vec = load <3 x i32>, ptr addrspace(1) %in, align 4
@@ -772,9 +890,10 @@ define amdgpu_kernel void @copy_v3i32_align4(ptr addrspace(1) noalias %out, ptr
}
define amdgpu_kernel void @copy_v3i64_align4(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #0 {
-; CHECK-LABEL: @copy_v3i64_align4(
-; CHECK-NEXT: [[VEC:%.*]] = load <3 x i64>, ptr addrspace(1) [[IN:%.*]], align 4
-; CHECK-NEXT: store <3 x i64> [[VEC]], ptr addrspace(1) [[OUT:%.*]], align 32
+; CHECK-LABEL: define amdgpu_kernel void @copy_v3i64_align4(
+; CHECK-SAME: ptr addrspace(1) noalias [[OUT:%.*]], ptr addrspace(1) noalias [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[VEC:%.*]] = load <3 x i64>, ptr addrspace(1) [[IN]], align 4
+; CHECK-NEXT: store <3 x i64> [[VEC]], ptr addrspace(1) [[OUT]], align 32
; CHECK-NEXT: ret void
;
%vec = load <3 x i64>, ptr addrspace(1) %in, align 4
@@ -783,10 +902,11 @@ define amdgpu_kernel void @copy_v3i64_align4(ptr addrspace(1) noalias %out, ptr
}
define amdgpu_kernel void @copy_v3f32_align4(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #0 {
-; CHECK-LABEL: @copy_v3f32_align4(
-; CHECK-NEXT: [[VEC:%.*]] = load <3 x float>, ptr addrspace(1) [[IN:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @copy_v3f32_align4(
+; CHECK-SAME: ptr addrspace(1) noalias [[OUT:%.*]], ptr addrspace(1) noalias [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[VEC:%.*]] = load <3 x float>, ptr addrspace(1) [[IN]], align 4
; CHECK-NEXT: [[FADD:%.*]] = fadd <3 x float> [[VEC]], <float 1.000000e+00, float 2.000000e+00, float 4.000000e+00>
-; CHECK-NEXT: store <3 x float> [[FADD]], ptr addrspace(1) [[OUT:%.*]], align 16
+; CHECK-NEXT: store <3 x float> [[FADD]], ptr addrspace(1) [[OUT]], align 16
; CHECK-NEXT: ret void
;
%vec = load <3 x float>, ptr addrspace(1) %in, align 4
@@ -796,10 +916,11 @@ define amdgpu_kernel void @copy_v3f32_align4(ptr addrspace(1) noalias %out, ptr
}
define amdgpu_kernel void @copy_v3f64_align4(ptr addrspace(1) noalias %out, ptr addrspace(1) noalias %in) #0 {
-; CHECK-LABEL: @copy_v3f64_align4(
-; CHECK-NEXT: [[VEC:%.*]] = load <3 x double>, ptr addrspace(1) [[IN:%.*]], align 4
+; CHECK-LABEL: define amdgpu_kernel void @copy_v3f64_align4(
+; CHECK-SAME: ptr addrspace(1) noalias [[OUT:%.*]], ptr addrspace(1) noalias [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[VEC:%.*]] = load <3 x double>, ptr addrspace(1) [[IN]], align 4
; CHECK-NEXT: [[FADD:%.*]] = fadd <3 x double> [[VEC]], <double 1.000000e+00, double 2.000000e+00, double 4.000000e+00>
-; CHECK-NEXT: store <3 x double> [[FADD]], ptr addrspace(1) [[OUT:%.*]], align 32
+; CHECK-NEXT: store <3 x double> [[FADD]], ptr addrspace(1) [[OUT]], align 32
; CHECK-NEXT: ret void
;
%vec = load <3 x double>, ptr addrspace(1) %in, align 4
@@ -810,11 +931,12 @@ define amdgpu_kernel void @copy_v3f64_align4(ptr addrspace(1) noalias %out, ptr
; Verify that we no longer hit asserts for this test case. No change expected.
define amdgpu_kernel void @copy_vec_of_ptrs(ptr addrspace(1) %out,
-; CHECK-LABEL: @copy_vec_of_ptrs(
-; CHECK-NEXT: [[IN_GEP_1:%.*]] = getelementptr <2 x ptr>, ptr addrspace(1) [[IN:%.*]], i32 1
+; CHECK-LABEL: define amdgpu_kernel void @copy_vec_of_ptrs(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[IN_GEP_1:%.*]] = getelementptr <2 x ptr>, ptr addrspace(1) [[IN]], i32 1
; CHECK-NEXT: [[VEC1:%.*]] = load <2 x ptr>, ptr addrspace(1) [[IN_GEP_1]], align 16
; CHECK-NEXT: [[VEC2:%.*]] = load <2 x ptr>, ptr addrspace(1) [[IN]], align 4
-; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr <2 x ptr>, ptr addrspace(1) [[OUT:%.*]], i32 1
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr <2 x ptr>, ptr addrspace(1) [[OUT]], i32 1
; CHECK-NEXT: store <2 x ptr> [[VEC1]], ptr addrspace(1) [[OUT_GEP_1]], align 16
; CHECK-NEXT: store <2 x ptr> [[VEC2]], ptr addrspace(1) [[OUT]], align 4
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors-complex.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors-complex.ll
index 5c331a9c9fe39..0a62a26d1c980 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors-complex.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors-complex.ll
@@ -5,21 +5,47 @@ define void @merge_i32_v2i16_f32_v4i8(ptr addrspace(1) %ptr1, ptr addrspace(2) %
; CHECK-LABEL: define void @merge_i32_v2i16_f32_v4i8(
; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR1]], i64 0
-; CHECK-NEXT: [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[GEP1]], align 4
-; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <2 x i16>, ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT: [[LOAD2:%.*]] = load <2 x i16>, ptr addrspace(1) [[GEP2]], align 4
-; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[PTR1]], i64 2
-; CHECK-NEXT: [[LOAD3:%.*]] = load float, ptr addrspace(1) [[GEP3]], align 4
-; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds <4 x i8>, ptr addrspace(1) [[PTR1]], i64 3
-; CHECK-NEXT: [[LOAD4:%.*]] = load <4 x i8>, ptr addrspace(1) [[GEP4]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <16 x b8>, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT: [[LOAD11:%.*]] = shufflevector <16 x b8> [[TMP1]], <16 x b8> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <4 x b8> [[LOAD11]] to i32
+; CHECK-NEXT: [[LOAD22:%.*]] = shufflevector <16 x b8> [[TMP1]], <16 x b8> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <4 x b8> [[LOAD22]] to <2 x i16>
+; CHECK-NEXT: [[LOAD33:%.*]] = shufflevector <16 x b8> [[TMP1]], <16 x b8> poison, <4 x i32> <i32 8, i32 9, i32 10, i32 11>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <4 x b8> [[LOAD33]] to float
+; CHECK-NEXT: [[LOAD44:%.*]] = shufflevector <16 x b8> [[TMP1]], <16 x b8> poison, <4 x i32> <i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <4 x b8> [[LOAD44]] to <4 x i8>
; CHECK-NEXT: [[STORE_GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 0
-; CHECK-NEXT: store i32 [[LOAD1]], ptr addrspace(2) [[STORE_GEP1]], align 4
-; CHECK-NEXT: [[STORE_GEP2:%.*]] = getelementptr inbounds <2 x i16>, ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT: store <2 x i16> [[LOAD2]], ptr addrspace(2) [[STORE_GEP2]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[TMP2]] to <2 x b16>
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x b16> [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x b16> poison, b16 [[TMP7]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x b16> [[TMP6]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x b16> [[TMP8]], b16 [[TMP9]], i32 1
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast <2 x i16> [[TMP3]] to <2 x b16>
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x b16> [[TMP11]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x b16> [[TMP10]], b16 [[TMP12]], i32 2
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x b16> [[TMP11]], i32 1
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x b16> [[TMP13]], b16 [[TMP14]], i32 3
+; CHECK-NEXT: store <4 x b16> [[TMP15]], ptr addrspace(2) [[STORE_GEP1]], align 4
; CHECK-NEXT: [[STORE_GEP3:%.*]] = getelementptr inbounds float, ptr addrspace(2) [[PTR2]], i64 2
-; CHECK-NEXT: store float [[LOAD3]], ptr addrspace(2) [[STORE_GEP3]], align 4
-; CHECK-NEXT: [[STORE_GEP4:%.*]] = getelementptr inbounds <4 x i8>, ptr addrspace(2) [[PTR2]], i64 3
-; CHECK-NEXT: store <4 x i8> [[LOAD4]], ptr addrspace(2) [[STORE_GEP4]], align 4
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast float [[TMP4]] to <4 x b8>
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <4 x b8> [[TMP16]], i32 0
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <8 x b8> poison, b8 [[TMP17]], i32 0
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x b8> [[TMP16]], i32 1
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <8 x b8> [[TMP18]], b8 [[TMP19]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x b8> [[TMP16]], i32 2
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <8 x b8> [[TMP20]], b8 [[TMP21]], i32 2
+; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x b8> [[TMP16]], i32 3
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <8 x b8> [[TMP22]], b8 [[TMP23]], i32 3
+; CHECK-NEXT: [[TMP25:%.*]] = bitcast <4 x i8> [[TMP5]] to <4 x b8>
+; CHECK-NEXT: [[TMP26:%.*]] = extractelement <4 x b8> [[TMP25]], i32 0
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <8 x b8> [[TMP24]], b8 [[TMP26]], i32 4
+; CHECK-NEXT: [[TMP28:%.*]] = extractelement <4 x b8> [[TMP25]], i32 1
+; CHECK-NEXT: [[TMP29:%.*]] = insertelement <8 x b8> [[TMP27]], b8 [[TMP28]], i32 5
+; CHECK-NEXT: [[TMP30:%.*]] = extractelement <4 x b8> [[TMP25]], i32 2
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <8 x b8> [[TMP29]], b8 [[TMP30]], i32 6
+; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x b8> [[TMP25]], i32 3
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <8 x b8> [[TMP31]], b8 [[TMP32]], i32 7
+; CHECK-NEXT: store <8 x b8> [[TMP33]], ptr addrspace(2) [[STORE_GEP3]], align 4
; CHECK-NEXT: ret void
;
%gep1 = getelementptr inbounds i32, ptr addrspace(1) %ptr1, i64 0
@@ -45,13 +71,23 @@ define void @merge_f32_v2f16_type(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2
; CHECK-LABEL: define void @merge_f32_v2f16_type(
; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds float, ptr addrspace(1) [[PTR1]], i64 0
-; CHECK-NEXT: [[LOAD1:%.*]] = load float, ptr addrspace(1) [[GEP1]], align 4
-; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds <2 x half>, ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT: [[LOAD2:%.*]] = load <2 x half>, ptr addrspace(1) [[GEP2]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b16>, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT: [[LOAD11:%.*]] = shufflevector <4 x b16> [[TMP1]], <4 x b16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b16> [[LOAD11]] to float
+; CHECK-NEXT: [[LOAD22:%.*]] = shufflevector <4 x b16> [[TMP1]], <4 x b16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b16> [[LOAD22]] to <2 x half>
; CHECK-NEXT: [[STORE_GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 0
-; CHECK-NEXT: store float [[LOAD1]], ptr addrspace(2) [[STORE_GEP1]], align 4
-; CHECK-NEXT: [[STORE_GEP2:%.*]] = getelementptr inbounds <2 x half>, ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT: store <2 x half> [[LOAD2]], ptr addrspace(2) [[STORE_GEP2]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast float [[TMP2]] to <2 x b16>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x b16> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x b16> poison, b16 [[TMP5]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x b16> [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x b16> [[TMP6]], b16 [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast <2 x half> [[TMP3]] to <2 x b16>
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x b16> [[TMP9]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x b16> [[TMP8]], b16 [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x b16> [[TMP9]], i32 1
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x b16> [[TMP11]], b16 [[TMP12]], i32 3
+; CHECK-NEXT: store <4 x b16> [[TMP13]], ptr addrspace(2) [[STORE_GEP1]], align 4
; CHECK-NEXT: ret void
;
%gep1 = getelementptr inbounds float, ptr addrspace(1) %ptr1, i64 0
@@ -93,9 +129,11 @@ define void @no_merge_mixed_ptr_addrspaces(ptr addrspace(1) %ptr1, ptr addrspace
; CHECK-LABEL: define void @no_merge_mixed_ptr_addrspaces(
; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) [[PTR1]], i64 0
-; CHECK-NEXT: [[LOAD1:%.*]] = load ptr addrspace(1), ptr addrspace(1) [[GEP1]], align 4
-; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds ptr addrspace(2), ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT: [[LOAD2:%.*]] = load ptr addrspace(2), ptr addrspace(1) [[GEP2]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT: [[LOAD11:%.*]] = shufflevector <2 x b32> [[TMP1]], <2 x b32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[LOAD1:%.*]] = bitcast <2 x b32> [[LOAD11]] to ptr addrspace(1)
+; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[LOAD2:%.*]] = bitcast b32 [[LOAD22]] to ptr addrspace(2)
; CHECK-NEXT: [[STORE_GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 0
; CHECK-NEXT: store ptr addrspace(1) [[LOAD1]], ptr addrspace(2) [[STORE_GEP1]], align 4
; CHECK-NEXT: [[STORE_GEP2:%.*]] = getelementptr inbounds ptr addrspace(2), ptr addrspace(2) [[PTR2]], i64 1
@@ -118,15 +156,17 @@ define void @merge_i32_p3(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
; CHECK-LABEL: define void @merge_i32_p3(
; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR1]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[GEP1]], align 4
-; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i32 [[LOAD22]] to ptr addrspace(3)
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LOAD11]] to i32
+; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LOAD22]] to ptr addrspace(3)
; CHECK-NEXT: [[STORE_GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> poison, i32 [[LOAD11]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = ptrtoint ptr addrspace(3) [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> [[TMP3]], i32 [[TMP4]], i32 1
-; CHECK-NEXT: store <2 x i32> [[TMP5]], ptr addrspace(2) [[STORE_GEP1]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x b32> poison, b32 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast ptr addrspace(3) [[TMP3]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x b32> [[TMP5]], b32 [[TMP6]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP7]], ptr addrspace(2) [[STORE_GEP1]], align 4
; CHECK-NEXT: ret void
;
%gep1 = getelementptr inbounds i32, ptr addrspace(1) %ptr1, i64 0
@@ -149,11 +189,11 @@ define void @split_i32_p1_p1(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR1]], i64 0
; CHECK-NEXT: [[LOAD1:%.*]] = load i32, ptr addrspace(1) [[GEP1]], align 8
; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr addrspace(1) [[GEP2]], align 8
-; CHECK-NEXT: [[LOAD21:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[LOAD21]] to ptr addrspace(1)
-; CHECK-NEXT: [[LOAD32:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[LOAD32]] to ptr addrspace(1)
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b64>, ptr addrspace(1) [[GEP2]], align 8
+; CHECK-NEXT: [[LOAD21:%.*]] = extractelement <2 x b64> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[LOAD21]] to ptr addrspace(1)
+; CHECK-NEXT: [[LOAD32:%.*]] = extractelement <2 x b64> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b64 [[LOAD32]] to ptr addrspace(1)
; CHECK-NEXT: [[STORE_GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 0
; CHECK-NEXT: store i32 [[LOAD1]], ptr addrspace(2) [[STORE_GEP1]], align 8
; CHECK-NEXT: [[STORE_GEP2:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(2) [[PTR2]], i64 1
@@ -184,19 +224,21 @@ define void @split_i32_i32_p1_p1(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2)
; CHECK-LABEL: define void @split_i32_i32_p1_p1(
; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR1]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[GEP1]], align 8
-; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i64>, ptr addrspace(1) [[GEP3]], align 8
-; CHECK-NEXT: [[LOAD33:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[LOAD33]] to ptr addrspace(1)
-; CHECK-NEXT: [[LOAD44:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i64 [[LOAD44]] to ptr addrspace(1)
+; CHECK-NEXT: [[TMP1:%.*]] = load <6 x b32>, ptr addrspace(1) [[GEP1]], align 8
+; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <6 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[LOAD11:%.*]] = bitcast b32 [[LOAD12]] to i32
+; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <6 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[LOAD22:%.*]] = bitcast b32 [[LOAD23]] to i32
+; CHECK-NEXT: [[LOAD33:%.*]] = shufflevector <6 x b32> [[TMP1]], <6 x b32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b32> [[LOAD33]] to ptr addrspace(1)
+; CHECK-NEXT: [[LOAD44:%.*]] = shufflevector <6 x b32> [[TMP1]], <6 x b32> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x b32> [[LOAD44]] to ptr addrspace(1)
; CHECK-NEXT: [[STORE_GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[LOAD11]], i32 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[LOAD22]], i32 1
-; CHECK-NEXT: store <2 x i32> [[TMP6]], ptr addrspace(2) [[STORE_GEP1]], align 8
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[LOAD11]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x b32> poison, b32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i32 [[LOAD22]] to b32
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x b32> [[TMP7]], b32 [[TMP8]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP9]], ptr addrspace(2) [[STORE_GEP1]], align 8
; CHECK-NEXT: [[STORE_GEP3:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(2) [[PTR2]], i64 1
; CHECK-NEXT: store ptr addrspace(1) [[TMP3]], ptr addrspace(2) [[STORE_GEP3]], align 8
; CHECK-NEXT: [[STORE_GEP4:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(2) [[PTR2]], i64 2
@@ -229,15 +271,19 @@ define void @split_i32_i32_p1(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
; CHECK-LABEL: define void @split_i32_i32_p1(
; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PTR1]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[GEP1]], align 8
-; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT: [[LOAD3:%.*]] = load ptr addrspace(1), ptr addrspace(1) [[GEP3]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(1) [[GEP1]], align 8
+; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[LOAD11:%.*]] = bitcast b32 [[LOAD12]] to i32
+; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[LOAD22:%.*]] = bitcast b32 [[LOAD23]] to i32
+; CHECK-NEXT: [[LOAD33:%.*]] = shufflevector <4 x b32> [[TMP1]], <4 x b32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[LOAD3:%.*]] = bitcast <2 x b32> [[LOAD33]] to ptr addrspace(1)
; CHECK-NEXT: [[STORE_GEP1:%.*]] = getelementptr inbounds i32, ptr addrspace(2) [[PTR2]], i64 0
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i32> poison, i32 [[LOAD11]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i32> [[TMP2]], i32 [[LOAD22]], i32 1
-; CHECK-NEXT: store <2 x i32> [[TMP3]], ptr addrspace(2) [[STORE_GEP1]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i32 [[LOAD11]] to b32
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x b32> poison, b32 [[TMP5]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32 [[LOAD22]] to b32
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x b32> [[TMP6]], b32 [[TMP7]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP8]], ptr addrspace(2) [[STORE_GEP1]], align 8
; CHECK-NEXT: [[STORE_GEP3:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(2) [[PTR2]], i64 1
; CHECK-NEXT: store ptr addrspace(1) [[LOAD3]], ptr addrspace(2) [[STORE_GEP3]], align 8
; CHECK-NEXT: ret void
@@ -263,17 +309,17 @@ define void @no_split_p3_p3(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
; CHECK-LABEL: define void @no_split_p3_p3(
; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(1) [[PTR1]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[GEP1]], align 4
-; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i32 [[LOAD11]] to ptr addrspace(3)
-; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[LOAD22]] to ptr addrspace(3)
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LOAD11]] to ptr addrspace(3)
+; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LOAD22]] to ptr addrspace(3)
; CHECK-NEXT: [[STORE_GEP1:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(2) [[PTR2]], i64 0
-; CHECK-NEXT: [[TMP4:%.*]] = ptrtoint ptr addrspace(3) [[TMP2]] to i32
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i32> poison, i32 [[TMP4]], i32 0
-; CHECK-NEXT: [[TMP6:%.*]] = ptrtoint ptr addrspace(3) [[TMP3]] to i32
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x i32> [[TMP5]], i32 [[TMP6]], i32 1
-; CHECK-NEXT: store <2 x i32> [[TMP7]], ptr addrspace(2) [[STORE_GEP1]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast ptr addrspace(3) [[TMP2]] to b32
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x b32> poison, b32 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast ptr addrspace(3) [[TMP3]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x b32> [[TMP5]], b32 [[TMP6]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP7]], ptr addrspace(2) [[STORE_GEP1]], align 4
; CHECK-NEXT: ret void
;
%gep1 = getelementptr inbounds ptr addrspace(3), ptr addrspace(1) %ptr1, i64 0
@@ -294,27 +340,25 @@ define void @split_p1_p1_p3_p3(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
; CHECK-LABEL: define void @split_p1_p1_p3_p3(
; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) [[PTR1]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr addrspace(1) [[GEP1]], align 8
-; CHECK-NEXT: [[LOAD13:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[LOAD13]] to ptr addrspace(1)
-; CHECK-NEXT: [[LOAD24:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[LOAD24]] to ptr addrspace(1)
-; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(1) [[PTR1]], i64 4
-; CHECK-NEXT: [[TMP4:%.*]] = load <2 x i32>, ptr addrspace(1) [[GEP3]], align 4
-; CHECK-NEXT: [[LOAD31:%.*]] = extractelement <2 x i32> [[TMP4]], i32 0
-; CHECK-NEXT: [[TMP5:%.*]] = inttoptr i32 [[LOAD31]] to ptr addrspace(3)
-; CHECK-NEXT: [[LOAD42:%.*]] = extractelement <2 x i32> [[TMP4]], i32 1
-; CHECK-NEXT: [[TMP6:%.*]] = inttoptr i32 [[LOAD42]] to ptr addrspace(3)
+; CHECK-NEXT: [[TMP1:%.*]] = load <6 x b32>, ptr addrspace(1) [[GEP1]], align 8
+; CHECK-NEXT: [[LOAD11:%.*]] = shufflevector <6 x b32> [[TMP1]], <6 x b32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b32> [[LOAD11]] to ptr addrspace(1)
+; CHECK-NEXT: [[LOAD22:%.*]] = shufflevector <6 x b32> [[TMP1]], <6 x b32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b32> [[LOAD22]] to ptr addrspace(1)
+; CHECK-NEXT: [[LOAD33:%.*]] = extractelement <6 x b32> [[TMP1]], i32 4
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LOAD33]] to ptr addrspace(3)
+; CHECK-NEXT: [[LOAD44:%.*]] = extractelement <6 x b32> [[TMP1]], i32 5
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[LOAD44]] to ptr addrspace(3)
; CHECK-NEXT: [[STORE_GEP1:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(2) [[PTR2]], i64 0
; CHECK-NEXT: store ptr addrspace(1) [[TMP2]], ptr addrspace(2) [[STORE_GEP1]], align 8
; CHECK-NEXT: [[STORE_GEP2:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(2) [[PTR2]], i64 1
; CHECK-NEXT: store ptr addrspace(1) [[TMP3]], ptr addrspace(2) [[STORE_GEP2]], align 8
; CHECK-NEXT: [[STORE_GEP3:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(2) [[PTR2]], i64 4
-; CHECK-NEXT: [[TMP7:%.*]] = ptrtoint ptr addrspace(3) [[TMP5]] to i32
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP9:%.*]] = ptrtoint ptr addrspace(3) [[TMP6]] to i32
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP9]], i32 1
-; CHECK-NEXT: store <2 x i32> [[TMP10]], ptr addrspace(2) [[STORE_GEP3]], align 4
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast ptr addrspace(3) [[TMP5]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x b32> poison, b32 [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast ptr addrspace(3) [[TMP6]] to b32
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x b32> [[TMP7]], b32 [[TMP8]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP9]], ptr addrspace(2) [[STORE_GEP3]], align 4
; CHECK-NEXT: ret void
;
%gep1 = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) %ptr1, i64 0
@@ -342,11 +386,11 @@ define void @no_split_p1_p1(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2) {
; CHECK-LABEL: define void @no_split_p1_p1(
; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) [[PTR1]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr addrspace(1) [[GEP1]], align 8
-; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP2:%.*]] = inttoptr i64 [[LOAD11]] to ptr addrspace(1)
-; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i64 [[LOAD22]] to ptr addrspace(1)
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b64>, ptr addrspace(1) [[GEP1]], align 8
+; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x b64> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[LOAD11]] to ptr addrspace(1)
+; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x b64> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b64 [[LOAD22]] to ptr addrspace(1)
; CHECK-NEXT: [[STORE_GEP1:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(2) [[PTR2]], i64 0
; CHECK-NEXT: store ptr addrspace(1) [[TMP2]], ptr addrspace(2) [[STORE_GEP1]], align 8
; CHECK-NEXT: [[STORE_GEP2:%.*]] = getelementptr inbounds ptr addrspace(1), ptr addrspace(2) [[PTR2]], i64 1
@@ -371,25 +415,28 @@ define void @split_i16_i16_p3_p3(ptr addrspace(1) %ptr1, ptr addrspace(2) %ptr2)
; CHECK-LABEL: define void @split_i16_i16_p3_p3(
; CHECK-SAME: ptr addrspace(1) [[PTR1:%.*]], ptr addrspace(2) [[PTR2:%.*]]) {
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i16, ptr addrspace(1) [[PTR1]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i16>, ptr addrspace(1) [[GEP1]], align 4
-; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x i16> [[TMP1]], i32 0
-; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x i16> [[TMP1]], i32 1
-; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(1) [[PTR1]], i64 1
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr addrspace(1) [[GEP3]], align 4
-; CHECK-NEXT: [[LOAD33:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = inttoptr i32 [[LOAD33]] to ptr addrspace(3)
-; CHECK-NEXT: [[LOAD44:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = inttoptr i32 [[LOAD44]] to ptr addrspace(3)
+; CHECK-NEXT: [[TMP1:%.*]] = load <6 x b16>, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <6 x b16> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b16 [[LOAD11]] to i16
+; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <6 x b16> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b16 [[LOAD22]] to i16
+; CHECK-NEXT: [[LOAD33:%.*]] = shufflevector <6 x b16> [[TMP1]], <6 x b16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x b16> [[LOAD33]] to ptr addrspace(3)
+; CHECK-NEXT: [[LOAD44:%.*]] = shufflevector <6 x b16> [[TMP1]], <6 x b16> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x b16> [[LOAD44]] to ptr addrspace(3)
; CHECK-NEXT: [[STORE_GEP1:%.*]] = getelementptr inbounds i16, ptr addrspace(2) [[PTR2]], i64 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x i16> poison, i16 [[LOAD11]], i32 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x i16> [[TMP5]], i16 [[LOAD22]], i32 1
-; CHECK-NEXT: store <2 x i16> [[TMP6]], ptr addrspace(2) [[STORE_GEP1]], align 4
-; CHECK-NEXT: [[STORE_GEP3:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(2) [[PTR2]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = ptrtoint ptr addrspace(3) [[TMP3]] to i32
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x i32> poison, i32 [[TMP7]], i32 0
-; CHECK-NEXT: [[TMP9:%.*]] = ptrtoint ptr addrspace(3) [[TMP4]] to i32
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <2 x i32> [[TMP8]], i32 [[TMP9]], i32 1
-; CHECK-NEXT: store <2 x i32> [[TMP10]], ptr addrspace(2) [[STORE_GEP3]], align 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i16 [[TMP2]] to b16
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x b16> poison, b16 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast i16 [[TMP3]] to b16
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x b16> [[TMP7]], b16 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast ptr addrspace(3) [[TMP4]] to <2 x b16>
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x b16> [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x b16> [[TMP9]], b16 [[TMP11]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <2 x b16> [[TMP10]], i32 1
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x b16> [[TMP12]], b16 [[TMP13]], i32 3
+; CHECK-NEXT: store <4 x b16> [[TMP14]], ptr addrspace(2) [[STORE_GEP1]], align 4
+; CHECK-NEXT: [[STORE_GEP4:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(2) [[PTR2]], i64 2
+; CHECK-NEXT: store ptr addrspace(3) [[TMP5]], ptr addrspace(2) [[STORE_GEP4]], align 4
; CHECK-NEXT: ret void
;
%gep1 = getelementptr inbounds i16, ptr addrspace(1) %ptr1, i64 0
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
index a696a5421f606..7fe72abf33762 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/merge-vectors.ll
@@ -10,10 +10,12 @@ define amdgpu_kernel void @merge_v2i32_v2i32(ptr addrspace(1) nocapture %a, ptr
; CHECK-LABEL: define amdgpu_kernel void @merge_v2i32_v2i32(
; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(1) readonly captures(none) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr addrspace(1) [[B]], align 4
-; CHECK-NEXT: [[LD_C1:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = shufflevector <4 x i32> [[TMP0]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: store <4 x i32> zeroinitializer, ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b32>, ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: [[LD_C1:%.*]] = shufflevector <4 x b32> [[TMP0]], <4 x b32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x b32> [[LD_C1]] to <2 x i32>
+; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = shufflevector <4 x b32> [[TMP0]], <4 x b32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b32> [[LD_C_IDX_12]] to <2 x i32>
+; CHECK-NEXT: store <4 x b32> zeroinitializer, ptr addrspace(1) [[A]], align 4
; CHECK-NEXT: ret void
;
entry:
@@ -33,10 +35,12 @@ define amdgpu_kernel void @merge_v1i32_v1i32(ptr addrspace(1) nocapture %a, ptr
; CHECK-LABEL: define amdgpu_kernel void @merge_v1i32_v1i32(
; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(1) readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i32>, ptr addrspace(1) [[B]], align 4
-; CHECK-NEXT: [[LD_C1:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <1 x i32> zeroinitializer
-; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = shufflevector <2 x i32> [[TMP0]], <2 x i32> poison, <1 x i32> <i32 1>
-; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: [[LD_C1:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_C1]] to <1 x i32>
+; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_C_IDX_12]] to <1 x i32>
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(1) [[A]], align 4
; CHECK-NEXT: ret void
;
entry:
@@ -81,10 +85,12 @@ define amdgpu_kernel void @merge_v2i16_v2i16(ptr addrspace(1) nocapture %a, ptr
; CHECK-LABEL: define amdgpu_kernel void @merge_v2i16_v2i16(
; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(1) readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(1) [[B]], align 4
-; CHECK-NEXT: [[LD_C1:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: store <4 x i16> zeroinitializer, ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b16>, ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: [[LD_C1:%.*]] = shufflevector <4 x b16> [[TMP0]], <4 x b16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x b16> [[LD_C1]] to <2 x i16>
+; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = shufflevector <4 x b16> [[TMP0]], <4 x b16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b16> [[LD_C_IDX_12]] to <2 x i16>
+; CHECK-NEXT: store <4 x b16> zeroinitializer, ptr addrspace(1) [[A]], align 4
; CHECK-NEXT: ret void
;
entry:
@@ -104,10 +110,12 @@ define amdgpu_kernel void @merge_fat_ptrs(ptr addrspace(7) nocapture %a, ptr add
; CHECK-OOB-RELAXED-LABEL: define amdgpu_kernel void @merge_fat_ptrs(
; CHECK-OOB-RELAXED-SAME: ptr addrspace(7) captures(none) [[A:%.*]], ptr addrspace(7) readonly captures(none) [[B:%.*]]) #[[ATTR0]] {
; CHECK-OOB-RELAXED-NEXT: [[ENTRY:.*:]]
-; CHECK-OOB-RELAXED-NEXT: [[TMP0:%.*]] = load <4 x i16>, ptr addrspace(7) [[B]], align 4
-; CHECK-OOB-RELAXED-NEXT: [[LD_C1:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-OOB-RELAXED-NEXT: [[LD_C_IDX_12:%.*]] = shufflevector <4 x i16> [[TMP0]], <4 x i16> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-OOB-RELAXED-NEXT: store <4 x i16> zeroinitializer, ptr addrspace(7) [[A]], align 4
+; CHECK-OOB-RELAXED-NEXT: [[TMP0:%.*]] = load <4 x b16>, ptr addrspace(7) [[B]], align 4
+; CHECK-OOB-RELAXED-NEXT: [[LD_C1:%.*]] = shufflevector <4 x b16> [[TMP0]], <4 x b16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-OOB-RELAXED-NEXT: [[TMP1:%.*]] = bitcast <2 x b16> [[LD_C1]] to <2 x i16>
+; CHECK-OOB-RELAXED-NEXT: [[LD_C_IDX_12:%.*]] = shufflevector <4 x b16> [[TMP0]], <4 x b16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-OOB-RELAXED-NEXT: [[TMP2:%.*]] = bitcast <2 x b16> [[LD_C_IDX_12]] to <2 x i16>
+; CHECK-OOB-RELAXED-NEXT: store <4 x b16> zeroinitializer, ptr addrspace(7) [[A]], align 4
; CHECK-OOB-RELAXED-NEXT: ret void
;
; CHECK-OOB-STRICT-LABEL: define amdgpu_kernel void @merge_fat_ptrs(
@@ -134,14 +142,15 @@ entry:
ret void
}
-; Ideally this would be merged
define amdgpu_kernel void @merge_load_i32_v2i16(ptr addrspace(1) nocapture %a) #0 {
; CHECK-LABEL: define amdgpu_kernel void @merge_load_i32_v2i16(
; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[A_1:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[A]], i32 1
-; CHECK-NEXT: [[LD_0:%.*]] = load i32, ptr addrspace(1) [[A]], align 4
-; CHECK-NEXT: [[LD_1:%.*]] = load <2 x i16>, ptr addrspace(1) [[A_1]], align 4
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b16>, ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: [[LD_01:%.*]] = shufflevector <4 x b16> [[TMP0]], <4 x b16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x b16> [[LD_01]] to i32
+; CHECK-NEXT: [[LD_12:%.*]] = shufflevector <4 x b16> [[TMP0]], <4 x b16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b16> [[LD_12]] to <2 x i16>
; CHECK-NEXT: ret void
;
entry:
@@ -153,5 +162,25 @@ entry:
ret void
}
+define amdgpu_kernel void @merge_load_v2i32_i16(ptr addrspace(1) nocapture %a) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_v2i32_i16(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b16>, ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: [[LD_01:%.*]] = shufflevector <4 x b16> [[TMP0]], <4 x b16> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <4 x b16> [[LD_01]] to <2 x i32>
+; CHECK-NEXT: [[LD_12:%.*]] = extractelement <4 x b16> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b16 [[LD_12]] to i16
+; CHECK-NEXT: ret void
+;
+entry:
+ %a.1 = getelementptr inbounds i32, ptr addrspace(1) %a, i32 1
+
+ %ld.0 = load <2 x i32>, ptr addrspace(1) %a
+ %ld.1 = load i16, ptr addrspace(1) %a.1
+
+ ret void
+}
+
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll
index 688a25055d626..068113eb31202 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/missing-alignment.ll
@@ -9,9 +9,11 @@
define amdgpu_kernel void @load_keep_base_alignment_missing_align(ptr addrspace(1) %out) {
; CHECK-LABEL: @load_keep_base_alignment_missing_align(
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr inbounds [512 x float], ptr addrspace(3) @lds, i32 0, i32 11
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x float>, ptr addrspace(3) [[PTR0]], align 4
-; CHECK-NEXT: [[VAL01:%.*]] = extractelement <2 x float> [[TMP2]], i32 0
-; CHECK-NEXT: [[VAL12:%.*]] = extractelement <2 x float> [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(3) [[PTR0]], align 4
+; CHECK-NEXT: [[VAL1:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[VAL01:%.*]] = bitcast b32 [[VAL1]] to float
+; CHECK-NEXT: [[VAL13:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[VAL12:%.*]] = bitcast b32 [[VAL13]] to float
; CHECK-NEXT: [[ADD:%.*]] = fadd float [[VAL01]], [[VAL12]]
; CHECK-NEXT: store float [[ADD]], ptr addrspace(1) [[OUT:%.*]], align 4
; CHECK-NEXT: ret void
@@ -29,7 +31,7 @@ define amdgpu_kernel void @load_keep_base_alignment_missing_align(ptr addrspace(
define amdgpu_kernel void @store_keep_base_alignment_missing_align() {
; CHECK-LABEL: @store_keep_base_alignment_missing_align(
; CHECK-NEXT: [[ARRAYIDX0:%.*]] = getelementptr inbounds [512 x float], ptr addrspace(3) @lds, i32 0, i32 1
-; CHECK-NEXT: store <2 x float> zeroinitializer, ptr addrspace(3) [[ARRAYIDX0]], align 4
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(3) [[ARRAYIDX0]], align 4
; CHECK-NEXT: ret void
;
%arrayidx0 = getelementptr inbounds [512 x float], ptr addrspace(3) @lds, i32 0, i32 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/mixed-ptr-and-narrow-int.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/mixed-ptr-and-narrow-int.ll
new file mode 100644
index 0000000000000..ab7c90d7a1d2e
--- /dev/null
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/mixed-ptr-and-narrow-int.ll
@@ -0,0 +1,88 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S < %s | FileCheck %s
+
+
+define amdgpu_kernel void @load_i16_i16_ptr32_i16(ptr addrspace(3) nocapture readonly align 16 dereferenceable(16) %blob) {
+; CHECK-LABEL: define amdgpu_kernel void @load_i16_i16_ptr32_i16(
+; CHECK-SAME: ptr addrspace(3) readonly align 16 captures(none) dereferenceable(16) [[BLOB:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[O0:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[BLOB]], i64 0
+; CHECK-NEXT: [[O8:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[BLOB]], i64 8
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b16>, ptr addrspace(3) [[O0]], align 16
+; CHECK-NEXT: [[X01:%.*]] = extractelement <4 x b16> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b16 [[X01]] to i16
+; CHECK-NEXT: [[X12:%.*]] = extractelement <4 x b16> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b16 [[X12]] to i16
+; CHECK-NEXT: [[X23:%.*]] = shufflevector <4 x b16> [[TMP0]], <4 x b16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b16> [[X23]] to ptr addrspace(3)
+; CHECK-NEXT: [[X3:%.*]] = load i16, ptr addrspace(3) [[O8]], align 16
+; CHECK-NEXT: ret void
+;
+entry:
+ %o0 = getelementptr inbounds i8, ptr addrspace(3) %blob, i64 0
+ %o2 = getelementptr inbounds i8, ptr addrspace(3) %blob, i64 2
+ %o4 = getelementptr inbounds i8, ptr addrspace(3) %blob, i64 4
+ %o8 = getelementptr inbounds i8, ptr addrspace(3) %blob, i64 8
+ %x0 = load i16, ptr addrspace(3) %o0, align 16
+ %x1 = load i16, ptr addrspace(3) %o2, align 16
+ %x2 = load ptr addrspace(3), ptr addrspace(3) %o4, align 16
+ %x3 = load i16, ptr addrspace(3) %o8, align 16
+ ret void
+}
+
+
+define amdgpu_kernel void @store_i16_i16_ptr32_i16(ptr addrspace(3) nocapture align 16 dereferenceable(16) %blob, i16 %a, i16 %b, ptr addrspace(3) %p, i16 %c) {
+; CHECK-LABEL: define amdgpu_kernel void @store_i16_i16_ptr32_i16(
+; CHECK-SAME: ptr addrspace(3) align 16 captures(none) dereferenceable(16) [[BLOB:%.*]], i16 [[A:%.*]], i16 [[B:%.*]], ptr addrspace(3) [[P:%.*]], i16 [[C:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[O0:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[BLOB]], i64 0
+; CHECK-NEXT: [[O8:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[BLOB]], i64 8
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast i16 [[A]] to b16
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x b16> poison, b16 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i16 [[B]] to b16
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x b16> [[TMP1]], b16 [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast ptr addrspace(3) [[P]] to <2 x b16>
+; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x b16> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x b16> [[TMP3]], b16 [[TMP5]], i32 2
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x b16> [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x b16> [[TMP6]], b16 [[TMP7]], i32 3
+; CHECK-NEXT: store <4 x b16> [[TMP8]], ptr addrspace(3) [[O0]], align 16
+; CHECK-NEXT: store i16 [[C]], ptr addrspace(3) [[O8]], align 16
+; CHECK-NEXT: ret void
+;
+entry:
+ %o0 = getelementptr inbounds i8, ptr addrspace(3) %blob, i64 0
+ %o2 = getelementptr inbounds i8, ptr addrspace(3) %blob, i64 2
+ %o4 = getelementptr inbounds i8, ptr addrspace(3) %blob, i64 4
+ %o8 = getelementptr inbounds i8, ptr addrspace(3) %blob, i64 8
+ store i16 %a, ptr addrspace(3) %o0, align 16
+ store i16 %b, ptr addrspace(3) %o2, align 16
+ store ptr addrspace(3) %p, ptr addrspace(3) %o4, align 16
+ store i16 %c, ptr addrspace(3) %o8, align 16
+ ret void
+}
+
+
+define amdgpu_kernel void @load_i32_ptr64_addrspace1(ptr addrspace(1) nocapture readonly align 16 dereferenceable(16) %blob) {
+; CHECK-LABEL: define amdgpu_kernel void @load_i32_ptr64_addrspace1(
+; CHECK-SAME: ptr addrspace(1) readonly align 16 captures(none) dereferenceable(16) [[BLOB:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[G0:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[BLOB]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b32>, ptr addrspace(1) [[G0]], align 16
+; CHECK-NEXT: [[X01:%.*]] = extractelement <4 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[X01]] to i32
+; CHECK-NEXT: [[X12:%.*]] = shufflevector <4 x b32> [[TMP0]], <4 x b32> poison, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b32> [[X12]] to ptr addrspace(1)
+; CHECK-NEXT: [[X23:%.*]] = extractelement <4 x b32> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[X23]] to i32
+; CHECK-NEXT: ret void
+;
+entry:
+ %g0 = getelementptr inbounds i32, ptr addrspace(1) %blob, i64 0
+ %g1 = getelementptr inbounds i32, ptr addrspace(1) %blob, i64 1
+ %g2 = getelementptr inbounds i32, ptr addrspace(1) %blob, i64 3
+ %x0 = load i32, ptr addrspace(1) %g0, align 16
+ %x1 = load ptr addrspace(1), ptr addrspace(1) %g1, align 16
+ %x2 = load i32, ptr addrspace(1) %g2, align 16
+ ret void
+}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
index 57da5976b3cfa..1ed02f08e25f4 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/multiple_tails.ll
@@ -12,7 +12,7 @@ define amdgpu_kernel void @no_crash(i32 %arg) {
; GCN-NEXT: [[TEMP3:%.*]] = getelementptr [16384 x i32], ptr addrspace(3) @[[GLOB0:[0-9]+]], i32 0, i32 [[TEMP2]]
; GCN-NEXT: [[TEMP4:%.*]] = add i32 [[ARG]], 15
; GCN-NEXT: [[TEMP5:%.*]] = getelementptr [16384 x i32], ptr addrspace(3) @[[GLOB0]], i32 0, i32 [[TEMP4]]
-; GCN-NEXT: store <2 x i32> zeroinitializer, ptr addrspace(3) [[TEMP3]], align 4
+; GCN-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(3) [[TEMP3]], align 4
; GCN-NEXT: store i32 0, ptr addrspace(3) [[TEMP5]], align 4
; GCN-NEXT: store i32 0, ptr addrspace(3) [[TEMP5]], align 4
; GCN-NEXT: ret void
@@ -40,18 +40,26 @@ define amdgpu_kernel void @interleave_get_longest(i32 %arg) {
; GCN-NEXT: [[TEMP1:%.*]] = getelementptr [16384 x i32], ptr addrspace(3) @[[GLOB0]], i32 0, i32 [[ARG]]
; GCN-NEXT: [[TEMP2:%.*]] = getelementptr [16384 x i32], ptr addrspace(3) @[[GLOB0]], i32 0, i32 [[A1]]
; GCN-NEXT: [[TEMP4:%.*]] = getelementptr [16384 x i32], ptr addrspace(3) @[[GLOB0]], i32 0, i32 [[A3]]
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(3) [[TEMP1]], align 4
-; GCN-NEXT: [[L21:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; GCN-NEXT: [[L12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; GCN-NEXT: store <2 x i32> zeroinitializer, ptr addrspace(3) [[TEMP1]], align 4
-; GCN-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr addrspace(3) [[TEMP2]], align 4
-; GCN-NEXT: [[L33:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
-; GCN-NEXT: [[L44:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
-; GCN-NEXT: [[TMP3:%.*]] = load <2 x i32>, ptr addrspace(3) [[TEMP4]], align 4
-; GCN-NEXT: [[L55:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
-; GCN-NEXT: [[L66:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
-; GCN-NEXT: [[L77:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
-; GCN-NEXT: [[L88:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
+; GCN-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(3) [[TEMP1]], align 4
+; GCN-NEXT: [[L21:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; GCN-NEXT: [[TMP2:%.*]] = bitcast b32 [[L21]] to i32
+; GCN-NEXT: [[L12:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; GCN-NEXT: [[TMP3:%.*]] = bitcast b32 [[L12]] to i32
+; GCN-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(3) [[TEMP1]], align 4
+; GCN-NEXT: [[TMP4:%.*]] = load <2 x b32>, ptr addrspace(3) [[TEMP2]], align 4
+; GCN-NEXT: [[L33:%.*]] = extractelement <2 x b32> [[TMP4]], i32 0
+; GCN-NEXT: [[TMP5:%.*]] = bitcast b32 [[L33]] to i32
+; GCN-NEXT: [[L44:%.*]] = extractelement <2 x b32> [[TMP4]], i32 1
+; GCN-NEXT: [[TMP6:%.*]] = bitcast b32 [[L44]] to i32
+; GCN-NEXT: [[TMP7:%.*]] = load <2 x b32>, ptr addrspace(3) [[TEMP4]], align 4
+; GCN-NEXT: [[L55:%.*]] = extractelement <2 x b32> [[TMP7]], i32 0
+; GCN-NEXT: [[TMP8:%.*]] = bitcast b32 [[L55]] to i32
+; GCN-NEXT: [[L66:%.*]] = extractelement <2 x b32> [[TMP7]], i32 1
+; GCN-NEXT: [[TMP9:%.*]] = bitcast b32 [[L66]] to i32
+; GCN-NEXT: [[L77:%.*]] = extractelement <2 x b32> [[TMP7]], i32 1
+; GCN-NEXT: [[TMP10:%.*]] = bitcast b32 [[L77]] to i32
+; GCN-NEXT: [[L88:%.*]] = extractelement <2 x b32> [[TMP7]], i32 1
+; GCN-NEXT: [[TMP11:%.*]] = bitcast b32 [[L88]] to i32
; GCN-NEXT: ret void
;
%a1 = add i32 %arg, 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll
index 0b2e1791084dc..3f650a60c9c00 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/no-implicit-float.ll
@@ -1,11 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
-; CHECK-LABEL: @no_implicit_float(
-; CHECK: store i32
-; CHECK: store i32
-; CHECK: store i32
-; CHECK: store i32
define amdgpu_kernel void @no_implicit_float(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_implicit_float(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: [[OUT_GEP_2:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 2
+; CHECK-NEXT: [[OUT_GEP_3:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 3
+; CHECK-NEXT: store i32 123, ptr addrspace(1) [[OUT_GEP_1]], align 4
+; CHECK-NEXT: store i32 456, ptr addrspace(1) [[OUT_GEP_2]], align 4
+; CHECK-NEXT: store i32 333, ptr addrspace(1) [[OUT_GEP_3]], align 4
+; CHECK-NEXT: store i32 1234, ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
%out.gep.2 = getelementptr i32, ptr addrspace(1) %out, i32 2
%out.gep.3 = getelementptr i32, ptr addrspace(1) %out, i32 3
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/opaque_ptr.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/opaque_ptr.ll
index c62744082f8ec..cd42f50d8dd68 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/opaque_ptr.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/opaque_ptr.ll
@@ -9,9 +9,11 @@
define i64 @order() {
; CHECK-LABEL: @order(
; CHECK-NEXT: [[IDX0:%.*]] = getelementptr inbounds [[S:%.*]], ptr @S, i32 0, i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[IDX0]], align 8
-; CHECK-NEXT: [[L01:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b64>, ptr [[IDX0]], align 8
+; CHECK-NEXT: [[L1:%.*]] = extractelement <2 x b64> [[TMP1]], i32 0
+; CHECK-NEXT: [[L01:%.*]] = bitcast b64 [[L1]] to i64
+; CHECK-NEXT: [[L13:%.*]] = extractelement <2 x b64> [[TMP1]], i32 1
+; CHECK-NEXT: [[L12:%.*]] = bitcast b64 [[L13]] to i64
; CHECK-NEXT: [[ADD:%.*]] = add i64 [[L01]], [[L12]]
; CHECK-NEXT: ret i64 [[ADD]]
;
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll
index d40d2771af17d..f7fcde73adf6e 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/optnone.ll
@@ -1,9 +1,14 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
-; CHECK-LABEL: @optnone(
-; CHECK: store i32
-; CHECK: store i32
define amdgpu_kernel void @optnone(ptr addrspace(1) %out) noinline optnone {
+; CHECK-LABEL: define amdgpu_kernel void @optnone(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i32 123, ptr addrspace(1) [[OUT_GEP_1]], align 4
+; CHECK-NEXT: store i32 456, ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
store i32 123, ptr addrspace(1) %out.gep.1
@@ -11,9 +16,12 @@ define amdgpu_kernel void @optnone(ptr addrspace(1) %out) noinline optnone {
ret void
}
-; CHECK-LABEL: @do_opt(
-; CHECK: store <2 x i32>
define amdgpu_kernel void @do_opt(ptr addrspace(1) %out) {
+; CHECK-LABEL: define amdgpu_kernel void @do_opt(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: store <2 x b32> <b32 456, b32 123>, ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i32, ptr addrspace(1) %out, i32 1
store i32 123, ptr addrspace(1) %out.gep.1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
index 2076a9dd962fe..d80ee730c2446 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/pointer-elements.ll
@@ -1,13 +1,20 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
declare i32 @llvm.amdgcn.workitem.id.x() #1
-; CHECK-LABEL: @merge_v2p1i8(
-; CHECK: load <2 x i64>
-; CHECK: inttoptr i64 %{{[^ ]+}} to ptr addrspace(1)
-; CHECK: inttoptr i64 %{{[^ ]+}} to ptr addrspace(1)
-; CHECK: store <2 x i64> zeroinitializer
define amdgpu_kernel void @merge_v2p1i8(ptr addrspace(1) nocapture %a, ptr addrspace(1) nocapture readonly %b) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_v2p1i8(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(1) readonly captures(none) [[B:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b64>, ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: [[LD_C1:%.*]] = extractelement <2 x b64> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b64 [[LD_C1]] to ptr addrspace(1)
+; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = extractelement <2 x b64> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[LD_C_IDX_12]] to ptr addrspace(1)
+; CHECK-NEXT: store <2 x b64> zeroinitializer, ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) %a, i64 1
%b.1 = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) %b, i64 1
@@ -21,12 +28,18 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_v2p3i8(
-; CHECK: load <2 x i32>
-; CHECK: inttoptr i32 %{{[^ ]+}} to ptr addrspace(3)
-; CHECK: inttoptr i32 %{{[^ ]+}} to ptr addrspace(3)
-; CHECK: store <2 x i32> zeroinitializer
define amdgpu_kernel void @merge_v2p3i8(ptr addrspace(3) nocapture %a, ptr addrspace(3) nocapture readonly %b) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_v2p3i8(
+; CHECK-SAME: ptr addrspace(3) captures(none) [[A:%.*]], ptr addrspace(3) readonly captures(none) [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(3) [[B]], align 4
+; CHECK-NEXT: [[LD_C1:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_C1]] to ptr addrspace(3)
+; CHECK-NEXT: [[LD_C_IDX_12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_C_IDX_12]] to ptr addrspace(3)
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr addrspace(3) [[A]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) %a, i64 1
%b.1 = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) %b, i64 1
@@ -40,10 +53,22 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_ptr_i32(
-; CHECK: load <4 x i32>
-; CHECK: store <4 x i32>
define amdgpu_kernel void @merge_ptr_i32(ptr addrspace(3) nocapture %a, ptr addrspace(3) nocapture readonly %b) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_ptr_i32(
+; CHECK-SAME: ptr addrspace(3) captures(none) [[A:%.*]], ptr addrspace(3) readonly captures(none) [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_0:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) [[A]], i64 0
+; CHECK-NEXT: [[B_0:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) [[B]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b32>, ptr addrspace(3) [[B_0]], align 16
+; CHECK-NEXT: [[LD_01:%.*]] = extractelement <4 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_01]] to i32
+; CHECK-NEXT: [[LD_12:%.*]] = extractelement <4 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_12]] to ptr addrspace(3)
+; CHECK-NEXT: [[LD_23:%.*]] = shufflevector <4 x b32> [[TMP0]], <4 x b32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b32> [[LD_23]] to <2 x i32>
+; CHECK-NEXT: store <4 x b32> zeroinitializer, ptr addrspace(3) [[A_0]], align 16
+; CHECK-NEXT: ret void
+;
entry:
%a.0 = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) %a, i64 0
%a.1 = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) %a, i64 1
@@ -64,10 +89,22 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_ptr_i32_vec_first(
-; CHECK: load <4 x i32>
-; CHECK: store <4 x i32>
define amdgpu_kernel void @merge_ptr_i32_vec_first(ptr addrspace(3) nocapture %a, ptr addrspace(3) nocapture readonly %b) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_ptr_i32_vec_first(
+; CHECK-SAME: ptr addrspace(3) captures(none) [[A:%.*]], ptr addrspace(3) readonly captures(none) [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_0:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) [[A]], i64 0
+; CHECK-NEXT: [[B_0:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) [[B]], i64 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b32>, ptr addrspace(3) [[B_0]], align 16
+; CHECK-NEXT: [[LD_01:%.*]] = shufflevector <4 x b32> [[TMP0]], <4 x b32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x b32> [[LD_01]] to <2 x i32>
+; CHECK-NEXT: [[LD_12:%.*]] = extractelement <4 x b32> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_12]] to ptr addrspace(3)
+; CHECK-NEXT: [[LD_23:%.*]] = extractelement <4 x b32> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD_23]] to i32
+; CHECK-NEXT: store <4 x b32> zeroinitializer, ptr addrspace(3) [[A_0]], align 16
+; CHECK-NEXT: ret void
+;
entry:
%a.0 = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) %a, i64 0
%a.1 = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) %a, i64 2
@@ -88,11 +125,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_load_i64_ptr64(
-; CHECK: load <2 x i64>
-; CHECK: [[ELT1:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 1
-; CHECK: inttoptr i64 [[ELT1]] to ptr addrspace(1)
define amdgpu_kernel void @merge_load_i64_ptr64(ptr addrspace(1) nocapture %a) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_i64_ptr64(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b64>, ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: [[LD_01:%.*]] = extractelement <2 x b64> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b64 [[LD_01]] to i64
+; CHECK-NEXT: [[LD_12:%.*]] = extractelement <2 x b64> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[LD_12]] to ptr addrspace(1)
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds i64, ptr addrspace(1) %a, i64 1
@@ -102,11 +145,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_load_ptr64_i64(
-; CHECK: load <2 x i64>
-; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 0
-; CHECK: inttoptr i64 [[ELT0]] to ptr addrspace(1)
define amdgpu_kernel void @merge_load_ptr64_i64(ptr addrspace(1) nocapture %a) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_ptr64_i64(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b64>, ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: [[LD_01:%.*]] = extractelement <2 x b64> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b64 [[LD_01]] to ptr addrspace(1)
+; CHECK-NEXT: [[LD_12:%.*]] = extractelement <2 x b64> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[LD_12]] to i64
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds i64, ptr addrspace(1) %a, i64 1
@@ -116,11 +165,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_store_ptr64_i64(
-; CHECK: [[ELT0:%[^ ]+]] = ptrtoint ptr addrspace(1) %ptr0 to i64
-; CHECK: insertelement <2 x i64> poison, i64 [[ELT0]], i32 0
-; CHECK: store <2 x i64>
define amdgpu_kernel void @merge_store_ptr64_i64(ptr addrspace(1) nocapture %a, ptr addrspace(1) %ptr0, i64 %val1) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_ptr64_i64(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(1) [[PTR0:%.*]], i64 [[VAL1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast ptr addrspace(1) [[PTR0]] to b64
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x b64> poison, b64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i64 [[VAL1]] to b64
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x b64> [[TMP1]], b64 [[TMP2]], i32 1
+; CHECK-NEXT: store <2 x b64> [[TMP3]], ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds i64, ptr addrspace(1) %a, i64 1
@@ -131,11 +186,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_store_i64_ptr64(
-; CHECK: [[ELT1:%[^ ]+]] = ptrtoint ptr addrspace(1) %ptr1 to i64
-; CHECK: insertelement <2 x i64> %{{[^ ]+}}, i64 [[ELT1]], i32 1
-; CHECK: store <2 x i64>
define amdgpu_kernel void @merge_store_i64_ptr64(ptr addrspace(1) nocapture %a, i64 %val0, ptr addrspace(1) %ptr1) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_i64_ptr64(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], i64 [[VAL0:%.*]], ptr addrspace(1) [[PTR1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast i64 [[VAL0]] to b64
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x b64> poison, b64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast ptr addrspace(1) [[PTR1]] to b64
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x b64> [[TMP1]], b64 [[TMP2]], i32 1
+; CHECK-NEXT: store <2 x b64> [[TMP3]], ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) %a, i64 1
@@ -145,11 +206,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_load_i32_ptr32(
-; CHECK: load <2 x i32>
-; CHECK: [[ELT1:%[^ ]+]] = extractelement <2 x i32> %{{[^ ]+}}, i32 1
-; CHECK: inttoptr i32 [[ELT1]] to ptr addrspace(3)
define amdgpu_kernel void @merge_load_i32_ptr32(ptr addrspace(3) nocapture %a) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_i32_ptr32(
+; CHECK-SAME: ptr addrspace(3) captures(none) [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(3) [[A]], align 4
+; CHECK-NEXT: [[LD_01:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_01]] to i32
+; CHECK-NEXT: [[LD_12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_12]] to ptr addrspace(3)
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds i32, ptr addrspace(3) %a, i32 1
@@ -159,11 +226,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_load_ptr32_i32(
-; CHECK: load <2 x i32>
-; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i32> %{{[^ ]+}}, i32 0
-; CHECK: inttoptr i32 [[ELT0]] to ptr addrspace(3)
define amdgpu_kernel void @merge_load_ptr32_i32(ptr addrspace(3) nocapture %a) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_ptr32_i32(
+; CHECK-SAME: ptr addrspace(3) captures(none) [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(3) [[A]], align 4
+; CHECK-NEXT: [[LD_01:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_01]] to ptr addrspace(3)
+; CHECK-NEXT: [[LD_12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_12]] to i32
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds i32, ptr addrspace(3) %a, i32 1
@@ -173,11 +246,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_store_ptr32_i32(
-; CHECK: [[ELT0:%[^ ]+]] = ptrtoint ptr addrspace(3) %ptr0 to i32
-; CHECK: insertelement <2 x i32> poison, i32 [[ELT0]], i32 0
-; CHECK: store <2 x i32>
define amdgpu_kernel void @merge_store_ptr32_i32(ptr addrspace(3) nocapture %a, ptr addrspace(3) %ptr0, i32 %val1) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_ptr32_i32(
+; CHECK-SAME: ptr addrspace(3) captures(none) [[A:%.*]], ptr addrspace(3) [[PTR0:%.*]], i32 [[VAL1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast ptr addrspace(3) [[PTR0]] to b32
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x b32> poison, b32 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 [[VAL1]] to b32
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x b32> [[TMP1]], b32 [[TMP2]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP3]], ptr addrspace(3) [[A]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds i32, ptr addrspace(3) %a, i32 1
@@ -187,11 +266,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_store_i32_ptr32(
-; CHECK: [[ELT1:%[^ ]+]] = ptrtoint ptr addrspace(3) %ptr1 to i32
-; CHECK: insertelement <2 x i32> %{{[^ ]+}}, i32 [[ELT1]], i32 1
-; CHECK: store <2 x i32>
define amdgpu_kernel void @merge_store_i32_ptr32(ptr addrspace(3) nocapture %a, i32 %val0, ptr addrspace(3) %ptr1) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_i32_ptr32(
+; CHECK-SAME: ptr addrspace(3) captures(none) [[A:%.*]], i32 [[VAL0:%.*]], ptr addrspace(3) [[PTR1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast i32 [[VAL0]] to b32
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x b32> poison, b32 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast ptr addrspace(3) [[PTR1]] to b32
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x b32> [[TMP1]], b32 [[TMP2]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP3]], ptr addrspace(3) [[A]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds ptr addrspace(3), ptr addrspace(3) %a, i32 1
@@ -201,10 +286,15 @@ entry:
ret void
}
-; CHECK-LABEL: @no_merge_store_ptr32_i64(
-; CHECK: store ptr addrspace(3)
-; CHECK: store i64
define amdgpu_kernel void @no_merge_store_ptr32_i64(ptr addrspace(1) nocapture %a, ptr addrspace(3) %ptr0, i64 %val1) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_merge_store_ptr32_i64(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(3) [[PTR0:%.*]], i64 [[VAL1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_1:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[A]], i64 1
+; CHECK-NEXT: store ptr addrspace(3) [[PTR0]], ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: store i64 [[VAL1]], ptr addrspace(1) [[A_1]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds i64, ptr addrspace(1) %a, i64 1
@@ -215,10 +305,15 @@ entry:
ret void
}
-; CHECK-LABEL: @no_merge_store_i64_ptr32(
-; CHECK: store i64
-; CHECK: store ptr addrspace(3)
define amdgpu_kernel void @no_merge_store_i64_ptr32(ptr addrspace(1) nocapture %a, i64 %val0, ptr addrspace(3) %ptr1) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_merge_store_i64_ptr32(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], i64 [[VAL0:%.*]], ptr addrspace(3) [[PTR1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_1:%.*]] = getelementptr inbounds ptr addrspace(3), ptr addrspace(1) [[A]], i64 1
+; CHECK-NEXT: store i64 [[VAL0]], ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: store ptr addrspace(3) [[PTR1]], ptr addrspace(1) [[A_1]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds ptr addrspace(3), ptr addrspace(1) %a, i64 1
@@ -228,10 +323,17 @@ entry:
ret void
}
-; CHECK-LABEL: @no_merge_load_i64_ptr32(
-; CHECK: load i64,
-; CHECK: load ptr addrspace(3),
define amdgpu_kernel void @no_merge_load_i64_ptr32(ptr addrspace(1) nocapture %a) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_merge_load_i64_ptr32(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <3 x b32>, ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: [[LD_01:%.*]] = shufflevector <3 x b32> [[TMP0]], <3 x b32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast <2 x b32> [[LD_01]] to i64
+; CHECK-NEXT: [[LD_12:%.*]] = extractelement <3 x b32> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_12]] to ptr addrspace(3)
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds i64, ptr addrspace(1) %a, i64 1
@@ -241,10 +343,15 @@ entry:
ret void
}
-; CHECK-LABEL: @no_merge_load_ptr32_i64(
-; CHECK: load ptr addrspace(3),
-; CHECK: load i64,
define amdgpu_kernel void @no_merge_load_ptr32_i64(ptr addrspace(1) nocapture %a) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @no_merge_load_ptr32_i64(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_1:%.*]] = getelementptr inbounds i64, ptr addrspace(1) [[A]], i64 1
+; CHECK-NEXT: [[LD_0:%.*]] = load ptr addrspace(3), ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: [[LD_1:%.*]] = load i64, ptr addrspace(1) [[A_1]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds i64, ptr addrspace(1) %a, i64 1
@@ -255,12 +362,18 @@ entry:
}
; XXX - This isn't merged for some reason
-; CHECK-LABEL: @merge_v2p1i8_v2p1i8(
-; CHECK: load <2 x ptr addrspace(1)>
-; CHECK: load <2 x ptr addrspace(1)>
-; CHECK: store <2 x ptr addrspace(1)>
-; CHECK: store <2 x ptr addrspace(1)>
define amdgpu_kernel void @merge_v2p1i8_v2p1i8(ptr addrspace(1) nocapture noalias %a, ptr addrspace(1) nocapture readonly noalias %b) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_v2p1i8_v2p1i8(
+; CHECK-SAME: ptr addrspace(1) noalias captures(none) [[A:%.*]], ptr addrspace(1) noalias readonly captures(none) [[B:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_1:%.*]] = getelementptr inbounds <2 x ptr addrspace(1)>, ptr addrspace(1) [[A]], i64 1
+; CHECK-NEXT: [[B_1:%.*]] = getelementptr inbounds <2 x ptr addrspace(1)>, ptr addrspace(1) [[B]], i64 1
+; CHECK-NEXT: [[LD_C:%.*]] = load <2 x ptr addrspace(1)>, ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: [[LD_C_IDX_1:%.*]] = load <2 x ptr addrspace(1)>, ptr addrspace(1) [[B_1]], align 4
+; CHECK-NEXT: store <2 x ptr addrspace(1)> splat (ptr addrspace(1) null), ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: store <2 x ptr addrspace(1)> splat (ptr addrspace(1) null), ptr addrspace(1) [[A_1]], align 4
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds <2 x ptr addrspace(1)>, ptr addrspace(1) %a, i64 1
%b.1 = getelementptr inbounds <2 x ptr addrspace(1)>, ptr addrspace(1) %b, i64 1
@@ -273,13 +386,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_load_ptr64_f64(
-; CHECK: load <2 x i64>
-; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 0
-; CHECK: [[ELT0_INT:%[^ ]+]] = inttoptr i64 [[ELT0]] to ptr addrspace(1)
-; CHECK: [[ELT1_INT:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 1
-; CHECK: bitcast i64 [[ELT1_INT]] to double
define amdgpu_kernel void @merge_load_ptr64_f64(ptr addrspace(1) nocapture %a) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_ptr64_f64(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b64>, ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: [[LD_01:%.*]] = extractelement <2 x b64> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b64 [[LD_01]] to ptr addrspace(1)
+; CHECK-NEXT: [[LD_12:%.*]] = extractelement <2 x b64> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[LD_12]] to double
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds double, ptr addrspace(1) %a, i64 1
@@ -289,13 +406,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_load_f64_ptr64(
-; CHECK: load <2 x i64>
-; CHECK: [[ELT0:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 0
-; CHECK: bitcast i64 [[ELT0]] to double
-; CHECK: [[ELT1:%[^ ]+]] = extractelement <2 x i64> %{{[^ ]+}}, i32 1
-; CHECK: inttoptr i64 [[ELT1]] to ptr addrspace(1)
define amdgpu_kernel void @merge_load_f64_ptr64(ptr addrspace(1) nocapture %a) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_f64_ptr64(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b64>, ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: [[LD_01:%.*]] = extractelement <2 x b64> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b64 [[LD_01]] to double
+; CHECK-NEXT: [[LD_12:%.*]] = extractelement <2 x b64> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[LD_12]] to ptr addrspace(1)
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds double, ptr addrspace(1) %a, i64 1
@@ -305,13 +426,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_store_ptr64_f64(
-; CHECK: [[ELT0_INT:%[^ ]+]] = ptrtoint ptr addrspace(1) %ptr0 to i64
-; CHECK: insertelement <2 x i64> poison, i64 [[ELT0_INT]], i32 0
-; CHECK: [[ELT1_INT:%[^ ]+]] = bitcast double %val1 to i64
-; CHECK: insertelement <2 x i64> %{{[^ ]+}}, i64 [[ELT1_INT]], i32 1
-; CHECK: store <2 x i64>
define amdgpu_kernel void @merge_store_ptr64_f64(ptr addrspace(1) nocapture %a, ptr addrspace(1) %ptr0, double %val1) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_ptr64_f64(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], ptr addrspace(1) [[PTR0:%.*]], double [[VAL1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast ptr addrspace(1) [[PTR0]] to b64
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x b64> poison, b64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast double [[VAL1]] to b64
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x b64> [[TMP1]], b64 [[TMP2]], i32 1
+; CHECK-NEXT: store <2 x b64> [[TMP3]], ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds double, ptr addrspace(1) %a, i64 1
@@ -321,13 +446,17 @@ entry:
ret void
}
-; CHECK-LABEL: @merge_store_f64_ptr64(
-; CHECK: [[ELT0_INT:%[^ ]+]] = bitcast double %val0 to i64
-; CHECK: insertelement <2 x i64> poison, i64 [[ELT0_INT]], i32 0
-; CHECK: [[ELT1_INT:%[^ ]+]] = ptrtoint ptr addrspace(1) %ptr1 to i64
-; CHECK: insertelement <2 x i64> %{{[^ ]+}}, i64 [[ELT1_INT]], i32 1
-; CHECK: store <2 x i64>
define amdgpu_kernel void @merge_store_f64_ptr64(ptr addrspace(1) nocapture %a, double %val0, ptr addrspace(1) %ptr1) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_f64_ptr64(
+; CHECK-SAME: ptr addrspace(1) captures(none) [[A:%.*]], double [[VAL0:%.*]], ptr addrspace(1) [[PTR1:%.*]]) #[[ATTR1]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast double [[VAL0]] to b64
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x b64> poison, b64 [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast ptr addrspace(1) [[PTR1]] to b64
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x b64> [[TMP1]], b64 [[TMP2]], i32 1
+; CHECK-NEXT: store <2 x b64> [[TMP3]], ptr addrspace(1) [[A]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%a.1 = getelementptr inbounds ptr addrspace(1), ptr addrspace(1) %a, i64 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll
index eecc91648e751..e271b11858ae7 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects-inseltpoison.ll
@@ -1,8 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer,dce -S -o - %s | FileCheck %s
define void @base_case(i1 %cnd, ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %out) {
-; CHECK-LABEL: @base_case
-; CHECK: load <3 x i32>
+; CHECK-LABEL: define void @base_case(
+; CHECK-SAME: i1 [[CND:%.*]], ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SELECTED:%.*]] = select i1 [[CND]], ptr addrspace(1) [[A]], ptr addrspace(1) [[B]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <3 x b32>, ptr addrspace(1) [[SELECTED]], align 4
+; CHECK-NEXT: [[VAL01:%.*]] = extractelement <3 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[VAL01]] to i32
+; CHECK-NEXT: [[VAL12:%.*]] = extractelement <3 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[VAL12]] to i32
+; CHECK-NEXT: [[VAL23:%.*]] = extractelement <3 x b32> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[VAL23]] to i32
+; CHECK-NEXT: [[T0:%.*]] = insertelement <3 x i32> poison, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = insertelement <3 x i32> [[T0]], i32 [[TMP2]], i32 1
+; CHECK-NEXT: [[T2:%.*]] = insertelement <3 x i32> [[T1]], i32 [[TMP3]], i32 2
+; CHECK-NEXT: store <3 x i32> [[T2]], ptr addrspace(1) [[OUT]], align 16
+; CHECK-NEXT: ret void
+;
entry:
%gep1 = getelementptr inbounds i32, ptr addrspace(1) %a, i64 1
%gep2 = getelementptr inbounds i32, ptr addrspace(1) %a, i64 2
@@ -22,8 +38,29 @@ entry:
}
define void @scev_targeting_complex_case(i1 %cnd, ptr addrspace(1) %a, ptr addrspace(1) %b, i32 %base, ptr addrspace(1) %out) {
-; CHECK-LABEL: @scev_targeting_complex_case
-; CHECK: load <2 x i32>
+; CHECK-LABEL: define void @scev_targeting_complex_case(
+; CHECK-SAME: i1 [[CND:%.*]], ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], i32 [[BASE:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[BASE_X4:%.*]] = shl i32 [[BASE]], 2
+; CHECK-NEXT: [[BASE_X4_P1:%.*]] = add i32 [[BASE_X4]], 1
+; CHECK-NEXT: [[ZEXT_X4:%.*]] = zext i32 [[BASE_X4]] to i64
+; CHECK-NEXT: [[ZEXT_X4_P1:%.*]] = zext i32 [[BASE_X4_P1]] to i64
+; CHECK-NEXT: [[BASE_X16:%.*]] = mul i64 [[ZEXT_X4]], 4
+; CHECK-NEXT: [[BASE_X16_P4:%.*]] = shl i64 [[ZEXT_X4_P1]], 2
+; CHECK-NEXT: [[GEP_A_BASE_X16:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i64 [[BASE_X16]]
+; CHECK-NEXT: [[GEP_B_BASE_X16_P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i64 [[BASE_X16_P4]]
+; CHECK-NEXT: [[SELECTED_BASE_X16_P0_OR_4:%.*]] = select i1 [[CND]], ptr addrspace(1) [[GEP_A_BASE_X16]], ptr addrspace(1) [[GEP_B_BASE_X16_P4]]
+; CHECK-NEXT: [[SELECTED_BASE_X16_P40_OR_44:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[SELECTED_BASE_X16_P0_OR_4]], i64 10
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[SELECTED_BASE_X16_P40_OR_44]], align 4
+; CHECK-NEXT: [[VAL01:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[VAL01]] to i32
+; CHECK-NEXT: [[VAL12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[VAL12]] to i32
+; CHECK-NEXT: [[T0:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = insertelement <2 x i32> [[T0]], i32 [[TMP2]], i32 1
+; CHECK-NEXT: store <2 x i32> [[T1]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%base.x4 = shl i32 %base, 2
%base.x4.p1 = add i32 %base.x4, 1
@@ -54,8 +91,30 @@ entry:
}
define void @nested_selects(i1 %cnd0, i1 %cnd1, ptr addrspace(1) %a, ptr addrspace(1) %b, i32 %base, ptr addrspace(1) %out) {
-; CHECK-LABEL: @nested_selects
-; CHECK: load <2 x i32>
+; CHECK-LABEL: define void @nested_selects(
+; CHECK-SAME: i1 [[CND0:%.*]], i1 [[CND1:%.*]], ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], i32 [[BASE:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[BASE_P2:%.*]] = add i32 [[BASE]], 2
+; CHECK-NEXT: [[BASE_X4:%.*]] = mul i32 [[BASE]], 4
+; CHECK-NEXT: [[BASE_X4_P5:%.*]] = add i32 [[BASE_X4]], 5
+; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[BASE]] to i64
+; CHECK-NEXT: [[SEXT_P2:%.*]] = sext i32 [[BASE_P2]] to i64
+; CHECK-NEXT: [[SEXT_X4_P5:%.*]] = sext i32 [[BASE_X4_P5]] to i64
+; CHECK-NEXT: [[GEP_A_BASE:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[A]], i64 [[SEXT]]
+; CHECK-NEXT: [[GEP_A_BASE_P2:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[A]], i64 [[SEXT_P2]]
+; CHECK-NEXT: [[GEP_B_BASE_X4_P5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[A]], i64 [[SEXT_X4_P5]]
+; CHECK-NEXT: [[SELECTED_1_L:%.*]] = select i1 [[CND1]], ptr addrspace(1) [[GEP_A_BASE_P2]], ptr addrspace(1) [[GEP_B_BASE_X4_P5]]
+; CHECK-NEXT: [[SELECTED_0_L:%.*]] = select i1 [[CND0]], ptr addrspace(1) [[GEP_A_BASE]], ptr addrspace(1) [[SELECTED_1_L]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[SELECTED_0_L]], align 4
+; CHECK-NEXT: [[VAL01:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[VAL01]] to i32
+; CHECK-NEXT: [[VAL12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[VAL12]] to i32
+; CHECK-NEXT: [[T0:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = insertelement <2 x i32> [[T0]], i32 [[TMP2]], i32 1
+; CHECK-NEXT: store <2 x i32> [[T1]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%base.p1 = add nsw i32 %base, 1
%base.p2 = add i32 %base, 2
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll
index 2471b64b92754..e271b11858ae7 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/selects.ll
@@ -1,8 +1,24 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer,dce -S -o - %s | FileCheck %s
define void @base_case(i1 %cnd, ptr addrspace(1) %a, ptr addrspace(1) %b, ptr addrspace(1) %out) {
-; CHECK-LABEL: @base_case
-; CHECK: load <3 x i32>
+; CHECK-LABEL: define void @base_case(
+; CHECK-SAME: i1 [[CND:%.*]], ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[SELECTED:%.*]] = select i1 [[CND]], ptr addrspace(1) [[A]], ptr addrspace(1) [[B]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <3 x b32>, ptr addrspace(1) [[SELECTED]], align 4
+; CHECK-NEXT: [[VAL01:%.*]] = extractelement <3 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[VAL01]] to i32
+; CHECK-NEXT: [[VAL12:%.*]] = extractelement <3 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[VAL12]] to i32
+; CHECK-NEXT: [[VAL23:%.*]] = extractelement <3 x b32> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[VAL23]] to i32
+; CHECK-NEXT: [[T0:%.*]] = insertelement <3 x i32> poison, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = insertelement <3 x i32> [[T0]], i32 [[TMP2]], i32 1
+; CHECK-NEXT: [[T2:%.*]] = insertelement <3 x i32> [[T1]], i32 [[TMP3]], i32 2
+; CHECK-NEXT: store <3 x i32> [[T2]], ptr addrspace(1) [[OUT]], align 16
+; CHECK-NEXT: ret void
+;
entry:
%gep1 = getelementptr inbounds i32, ptr addrspace(1) %a, i64 1
%gep2 = getelementptr inbounds i32, ptr addrspace(1) %a, i64 2
@@ -14,7 +30,7 @@ entry:
%val0 = load i32, ptr addrspace(1) %selected, align 4
%val1 = load i32, ptr addrspace(1) %selected14, align 4
%val2 = load i32, ptr addrspace(1) %selected25, align 4
- %t0 = insertelement <3 x i32> undef, i32 %val0, i32 0
+ %t0 = insertelement <3 x i32> poison, i32 %val0, i32 0
%t1 = insertelement <3 x i32> %t0, i32 %val1, i32 1
%t2 = insertelement <3 x i32> %t1, i32 %val2, i32 2
store <3 x i32> %t2, ptr addrspace(1) %out
@@ -22,8 +38,29 @@ entry:
}
define void @scev_targeting_complex_case(i1 %cnd, ptr addrspace(1) %a, ptr addrspace(1) %b, i32 %base, ptr addrspace(1) %out) {
-; CHECK-LABEL: @scev_targeting_complex_case
-; CHECK: load <2 x i32>
+; CHECK-LABEL: define void @scev_targeting_complex_case(
+; CHECK-SAME: i1 [[CND:%.*]], ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], i32 [[BASE:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[BASE_X4:%.*]] = shl i32 [[BASE]], 2
+; CHECK-NEXT: [[BASE_X4_P1:%.*]] = add i32 [[BASE_X4]], 1
+; CHECK-NEXT: [[ZEXT_X4:%.*]] = zext i32 [[BASE_X4]] to i64
+; CHECK-NEXT: [[ZEXT_X4_P1:%.*]] = zext i32 [[BASE_X4_P1]] to i64
+; CHECK-NEXT: [[BASE_X16:%.*]] = mul i64 [[ZEXT_X4]], 4
+; CHECK-NEXT: [[BASE_X16_P4:%.*]] = shl i64 [[ZEXT_X4_P1]], 2
+; CHECK-NEXT: [[GEP_A_BASE_X16:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[A]], i64 [[BASE_X16]]
+; CHECK-NEXT: [[GEP_B_BASE_X16_P4:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[B]], i64 [[BASE_X16_P4]]
+; CHECK-NEXT: [[SELECTED_BASE_X16_P0_OR_4:%.*]] = select i1 [[CND]], ptr addrspace(1) [[GEP_A_BASE_X16]], ptr addrspace(1) [[GEP_B_BASE_X16_P4]]
+; CHECK-NEXT: [[SELECTED_BASE_X16_P40_OR_44:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[SELECTED_BASE_X16_P0_OR_4]], i64 10
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[SELECTED_BASE_X16_P40_OR_44]], align 4
+; CHECK-NEXT: [[VAL01:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[VAL01]] to i32
+; CHECK-NEXT: [[VAL12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[VAL12]] to i32
+; CHECK-NEXT: [[T0:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = insertelement <2 x i32> [[T0]], i32 [[TMP2]], i32 1
+; CHECK-NEXT: store <2 x i32> [[T1]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%base.x4 = shl i32 %base, 2
%base.x4.p1 = add i32 %base.x4, 1
@@ -47,15 +84,37 @@ entry:
%selected.base.x16.p44.or.48 = getelementptr inbounds i32, ptr addrspace(1) %gep.selected.base.x16.p8.or.12, i64 9
%val0 = load i32, ptr addrspace(1) %selected.base.x16.p40.or.44, align 4
%val1 = load i32, ptr addrspace(1) %selected.base.x16.p44.or.48, align 4
- %t0 = insertelement <2 x i32> undef, i32 %val0, i32 0
+ %t0 = insertelement <2 x i32> poison, i32 %val0, i32 0
%t1 = insertelement <2 x i32> %t0, i32 %val1, i32 1
store <2 x i32> %t1, ptr addrspace(1) %out
ret void
}
define void @nested_selects(i1 %cnd0, i1 %cnd1, ptr addrspace(1) %a, ptr addrspace(1) %b, i32 %base, ptr addrspace(1) %out) {
-; CHECK-LABEL: @nested_selects
-; CHECK: load <2 x i32>
+; CHECK-LABEL: define void @nested_selects(
+; CHECK-SAME: i1 [[CND0:%.*]], i1 [[CND1:%.*]], ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]], i32 [[BASE:%.*]], ptr addrspace(1) [[OUT:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[BASE_P2:%.*]] = add i32 [[BASE]], 2
+; CHECK-NEXT: [[BASE_X4:%.*]] = mul i32 [[BASE]], 4
+; CHECK-NEXT: [[BASE_X4_P5:%.*]] = add i32 [[BASE_X4]], 5
+; CHECK-NEXT: [[SEXT:%.*]] = sext i32 [[BASE]] to i64
+; CHECK-NEXT: [[SEXT_P2:%.*]] = sext i32 [[BASE_P2]] to i64
+; CHECK-NEXT: [[SEXT_X4_P5:%.*]] = sext i32 [[BASE_X4_P5]] to i64
+; CHECK-NEXT: [[GEP_A_BASE:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[A]], i64 [[SEXT]]
+; CHECK-NEXT: [[GEP_A_BASE_P2:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[A]], i64 [[SEXT_P2]]
+; CHECK-NEXT: [[GEP_B_BASE_X4_P5:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[A]], i64 [[SEXT_X4_P5]]
+; CHECK-NEXT: [[SELECTED_1_L:%.*]] = select i1 [[CND1]], ptr addrspace(1) [[GEP_A_BASE_P2]], ptr addrspace(1) [[GEP_B_BASE_X4_P5]]
+; CHECK-NEXT: [[SELECTED_0_L:%.*]] = select i1 [[CND0]], ptr addrspace(1) [[GEP_A_BASE]], ptr addrspace(1) [[SELECTED_1_L]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(1) [[SELECTED_0_L]], align 4
+; CHECK-NEXT: [[VAL01:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[VAL01]] to i32
+; CHECK-NEXT: [[VAL12:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[VAL12]] to i32
+; CHECK-NEXT: [[T0:%.*]] = insertelement <2 x i32> poison, i32 [[TMP1]], i32 0
+; CHECK-NEXT: [[T1:%.*]] = insertelement <2 x i32> [[T0]], i32 [[TMP2]], i32 1
+; CHECK-NEXT: store <2 x i32> [[T1]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%base.p1 = add nsw i32 %base, 1
%base.p2 = add i32 %base, 2
@@ -81,7 +140,7 @@ entry:
%selected.0.R = select i1 %cnd0, ptr addrspace(1) %gep.a.base.p1, ptr addrspace(1) %selected.1.R
%val0 = load i32, ptr addrspace(1) %selected.0.L, align 4
%val1 = load i32, ptr addrspace(1) %selected.0.R, align 4
- %t0 = insertelement <2 x i32> undef, i32 %val0, i32 0
+ %t0 = insertelement <2 x i32> poison, i32 %val0, i32 0
%t1 = insertelement <2 x i32> %t0, i32 %val1, i32 1
store <2 x i32> %t1, ptr addrspace(1) %out
ret void
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll
index 69fdf6d0f0130..feb42f1b222f8 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/store_with_aliasing_load.ll
@@ -1,15 +1,46 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --function store_vectorize_with_alias --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; Check that, in the presence of an aliasing load, the stores preceding the
; aliasing load are safe to vectorize.
-; CHECK-LABEL: store_vectorize_with_alias
-; CHECK: store <4 x float>
-; CHECK: load <4 x float>
-; CHECK: store <4 x float>
-
; Function Attrs: nounwind
define amdgpu_kernel void @store_vectorize_with_alias(ptr addrspace(1) %a, ptr addrspace(1) %b) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @store_vectorize_with_alias(
+; CHECK-SAME: ptr addrspace(1) [[A:%.*]], ptr addrspace(1) [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[LD:%.*]] = load float, ptr addrspace(1) [[B]], align 4
+; CHECK-NEXT: [[BC0:%.*]] = bitcast float [[LD]] to b32
+; CHECK-NEXT: [[VECA:%.*]] = insertelement <4 x b32> poison, b32 [[BC0]], i32 0
+; CHECK-NEXT: [[BC1:%.*]] = bitcast float [[LD]] to b32
+; CHECK-NEXT: [[VECB:%.*]] = insertelement <4 x b32> [[VECA]], b32 [[BC1]], i32 1
+; CHECK-NEXT: [[BC2:%.*]] = bitcast float [[LD]] to b32
+; CHECK-NEXT: [[VECC:%.*]] = insertelement <4 x b32> [[VECB]], b32 [[BC2]], i32 2
+; CHECK-NEXT: [[BC3:%.*]] = bitcast float [[LD]] to b32
+; CHECK-NEXT: [[VECD:%.*]] = insertelement <4 x b32> [[VECC]], b32 [[BC3]], i32 3
+; CHECK-NEXT: store <4 x b32> [[VECD]], ptr addrspace(1) [[A]], align 4
+; CHECK-NEXT: [[GEPB:%.*]] = getelementptr i8, ptr addrspace(1) [[B]], i64 16
+; CHECK-NEXT: [[VL:%.*]] = load <4 x b32>, ptr addrspace(1) [[GEPB]], align 4
+; CHECK-NEXT: [[EX0:%.*]] = extractelement <4 x b32> [[VL]], i32 0
+; CHECK-NEXT: [[F0:%.*]] = bitcast b32 [[EX0]] to float
+; CHECK-NEXT: [[EX1:%.*]] = extractelement <4 x b32> [[VL]], i32 1
+; CHECK-NEXT: [[F1:%.*]] = bitcast b32 [[EX1]] to float
+; CHECK-NEXT: [[EX2:%.*]] = extractelement <4 x b32> [[VL]], i32 2
+; CHECK-NEXT: [[F2:%.*]] = bitcast b32 [[EX2]] to float
+; CHECK-NEXT: [[EX3:%.*]] = extractelement <4 x b32> [[VL]], i32 3
+; CHECK-NEXT: [[F3:%.*]] = bitcast b32 [[EX3]] to float
+; CHECK-NEXT: [[GEPA:%.*]] = getelementptr i8, ptr addrspace(1) [[A]], i64 16
+; CHECK-NEXT: [[BC4:%.*]] = bitcast float [[F0]] to b32
+; CHECK-NEXT: [[VS0:%.*]] = insertelement <4 x b32> poison, b32 [[BC4]], i32 0
+; CHECK-NEXT: [[BC5:%.*]] = bitcast float [[F1]] to b32
+; CHECK-NEXT: [[VS1:%.*]] = insertelement <4 x b32> [[VS0]], b32 [[BC5]], i32 1
+; CHECK-NEXT: [[BC6:%.*]] = bitcast float [[F2]] to b32
+; CHECK-NEXT: [[VS2:%.*]] = insertelement <4 x b32> [[VS1]], b32 [[BC6]], i32 2
+; CHECK-NEXT: [[BC7:%.*]] = bitcast float [[F3]] to b32
+; CHECK-NEXT: [[VS3:%.*]] = insertelement <4 x b32> [[VS2]], b32 [[BC7]], i32 3
+; CHECK-NEXT: store <4 x b32> [[VS3]], ptr addrspace(1) [[GEPA]], align 4
+; CHECK-NEXT: ret void
+;
bb:
%tmp1 = load float, ptr addrspace(1) %b, align 4
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/unaligned-buffer.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/unaligned-buffer.ll
index 59c30e12868b8..2ac2657c5168a 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/unaligned-buffer.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/unaligned-buffer.ll
@@ -27,11 +27,15 @@ define amdgpu_kernel void @merge_align_4(ptr addrspace(7) captures(none) %p) #0
; OOB-RELAXED-SAME: ptr addrspace(7) captures(none) [[P:%.*]]) {
; OOB-RELAXED-NEXT: [[ENTRY:.*:]]
; OOB-RELAXED-NEXT: [[GEP_M8:%.*]] = getelementptr i8, ptr addrspace(7) [[P]], i32 -8
-; OOB-RELAXED-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr addrspace(7) [[GEP_M8]], align 4
-; OOB-RELAXED-NEXT: [[LD_M81:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
-; OOB-RELAXED-NEXT: [[LD_M42:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
-; OOB-RELAXED-NEXT: [[LD_03:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
-; OOB-RELAXED-NEXT: [[LD_44:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
+; OOB-RELAXED-NEXT: [[TMP0:%.*]] = load <4 x b32>, ptr addrspace(7) [[GEP_M8]], align 4
+; OOB-RELAXED-NEXT: [[LD_M81:%.*]] = extractelement <4 x b32> [[TMP0]], i32 0
+; OOB-RELAXED-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_M81]] to i32
+; OOB-RELAXED-NEXT: [[LD_M42:%.*]] = extractelement <4 x b32> [[TMP0]], i32 1
+; OOB-RELAXED-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_M42]] to i32
+; OOB-RELAXED-NEXT: [[LD_03:%.*]] = extractelement <4 x b32> [[TMP0]], i32 2
+; OOB-RELAXED-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD_03]] to i32
+; OOB-RELAXED-NEXT: [[LD_44:%.*]] = extractelement <4 x b32> [[TMP0]], i32 3
+; OOB-RELAXED-NEXT: [[TMP4:%.*]] = bitcast b32 [[LD_44]] to i32
; OOB-RELAXED-NEXT: ret void
;
entry:
@@ -54,22 +58,30 @@ define amdgpu_kernel void @merge_align_16(ptr addrspace(7) captures(none) %p) #0
; OOB-STRICT-SAME: ptr addrspace(7) captures(none) [[P:%.*]]) {
; OOB-STRICT-NEXT: [[ENTRY:.*:]]
; OOB-STRICT-NEXT: [[GEP_M8:%.*]] = getelementptr i8, ptr addrspace(7) [[P]], i32 -8
-; OOB-STRICT-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr addrspace(7) [[GEP_M8]], align 16
-; OOB-STRICT-NEXT: [[LD_M81:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
-; OOB-STRICT-NEXT: [[LD_M42:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
-; OOB-STRICT-NEXT: [[LD_03:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
-; OOB-STRICT-NEXT: [[LD_44:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
+; OOB-STRICT-NEXT: [[TMP0:%.*]] = load <4 x b32>, ptr addrspace(7) [[GEP_M8]], align 16
+; OOB-STRICT-NEXT: [[LD_M81:%.*]] = extractelement <4 x b32> [[TMP0]], i32 0
+; OOB-STRICT-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_M81]] to i32
+; OOB-STRICT-NEXT: [[LD_M42:%.*]] = extractelement <4 x b32> [[TMP0]], i32 1
+; OOB-STRICT-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_M42]] to i32
+; OOB-STRICT-NEXT: [[LD_03:%.*]] = extractelement <4 x b32> [[TMP0]], i32 2
+; OOB-STRICT-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD_03]] to i32
+; OOB-STRICT-NEXT: [[LD_44:%.*]] = extractelement <4 x b32> [[TMP0]], i32 3
+; OOB-STRICT-NEXT: [[TMP4:%.*]] = bitcast b32 [[LD_44]] to i32
; OOB-STRICT-NEXT: ret void
;
; OOB-RELAXED-LABEL: define amdgpu_kernel void @merge_align_16(
; OOB-RELAXED-SAME: ptr addrspace(7) captures(none) [[P:%.*]]) {
; OOB-RELAXED-NEXT: [[ENTRY:.*:]]
; OOB-RELAXED-NEXT: [[GEP_M8:%.*]] = getelementptr i8, ptr addrspace(7) [[P]], i32 -8
-; OOB-RELAXED-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr addrspace(7) [[GEP_M8]], align 16
-; OOB-RELAXED-NEXT: [[LD_M81:%.*]] = extractelement <4 x i32> [[TMP0]], i32 0
-; OOB-RELAXED-NEXT: [[LD_M42:%.*]] = extractelement <4 x i32> [[TMP0]], i32 1
-; OOB-RELAXED-NEXT: [[LD_03:%.*]] = extractelement <4 x i32> [[TMP0]], i32 2
-; OOB-RELAXED-NEXT: [[LD_44:%.*]] = extractelement <4 x i32> [[TMP0]], i32 3
+; OOB-RELAXED-NEXT: [[TMP0:%.*]] = load <4 x b32>, ptr addrspace(7) [[GEP_M8]], align 16
+; OOB-RELAXED-NEXT: [[LD_M81:%.*]] = extractelement <4 x b32> [[TMP0]], i32 0
+; OOB-RELAXED-NEXT: [[TMP1:%.*]] = bitcast b32 [[LD_M81]] to i32
+; OOB-RELAXED-NEXT: [[LD_M42:%.*]] = extractelement <4 x b32> [[TMP0]], i32 1
+; OOB-RELAXED-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD_M42]] to i32
+; OOB-RELAXED-NEXT: [[LD_03:%.*]] = extractelement <4 x b32> [[TMP0]], i32 2
+; OOB-RELAXED-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD_03]] to i32
+; OOB-RELAXED-NEXT: [[LD_44:%.*]] = extractelement <4 x b32> [[TMP0]], i32 3
+; OOB-RELAXED-NEXT: [[TMP4:%.*]] = bitcast b32 [[LD_44]] to i32
; OOB-RELAXED-NEXT: ret void
;
entry:
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll
index f9411c0129d4d..9932da4f11cc5 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vect-ptr-ptr-size-mismatch.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=load-store-vectorizer -S < %s | FileCheck %s
target datalayout = "e-p:64:64-p1:64:64-p5:32:32"
@@ -6,39 +7,65 @@ target datalayout = "e-p:64:64-p1:64:64-p5:32:32"
; pointer in address space 0 it was cast to caused below test to crash.
; The p5:32:32 portion of the data layout is critical for the test.
-; CHECK-LABEL: @cast_to_ptr
-; CHECK: store ptr undef, ptr %tmp9, align 8
-; CHECK: store ptr undef, ptr %tmp7, align 8
define void @cast_to_ptr() {
+; CHECK-LABEL: define void @cast_to_ptr() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ASCAST:%.*]] = addrspacecast ptr addrspace(5) null to ptr
+; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i32 poison, 0
+; CHECK-NEXT: [[TMP6:%.*]] = select i1 false, ptr poison, ptr poison
+; CHECK-NEXT: [[TMP7:%.*]] = select i1 [[TMP4]], ptr null, ptr [[TMP6]]
+; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP4]], ptr [[ASCAST]], ptr null
+; CHECK-NEXT: store ptr poison, ptr [[TMP9]], align 8
+; CHECK-NEXT: store ptr poison, ptr [[TMP7]], align 8
+; CHECK-NEXT: unreachable
+;
entry:
%ascast = addrspacecast ptr addrspace(5) zeroinitializer to ptr
- %tmp4 = icmp eq i32 undef, 0
- %tmp6 = select i1 false, ptr undef, ptr undef
+ %tmp4 = icmp eq i32 poison, 0
+ %tmp6 = select i1 false, ptr poison, ptr poison
%tmp7 = select i1 %tmp4, ptr null, ptr %tmp6
%tmp9 = select i1 %tmp4, ptr %ascast, ptr null
- store ptr undef, ptr %tmp9, align 8
- store ptr undef, ptr %tmp7, align 8
+ store ptr poison, ptr %tmp9, align 8
+ store ptr poison, ptr %tmp7, align 8
unreachable
}
-; CHECK-LABEL: @cast_to_cast
-; CHECK: load i64
-; CHECK-NEXT: inttoptr i64
-; CHECK-NEXT: inttoptr i64
define void @cast_to_cast() {
+; CHECK-LABEL: define void @cast_to_cast() {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_ASCAST:%.*]] = addrspacecast ptr addrspace(5) poison to ptr
+; CHECK-NEXT: [[TMP1:%.*]] = select i1 false, ptr [[A_ASCAST]], ptr poison
+; CHECK-NEXT: [[TMP0:%.*]] = load b64, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b64 [[TMP0]] to ptr
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[TMP0]] to ptr
+; CHECK-NEXT: unreachable
+;
entry:
- %a.ascast = addrspacecast ptr addrspace(5) undef to ptr
+ %a.ascast = addrspacecast ptr addrspace(5) poison to ptr
%b.ascast = addrspacecast ptr addrspace(5) zeroinitializer to ptr
- %tmp1 = select i1 false, ptr %a.ascast, ptr undef
- %tmp3 = select i1 false, ptr %b.ascast, ptr undef
+ %tmp1 = select i1 false, ptr %a.ascast, ptr poison
+ %tmp3 = select i1 false, ptr %b.ascast, ptr poison
%tmp4 = load ptr, ptr %tmp1, align 8
%tmp5 = load ptr, ptr %tmp3, align 8
unreachable
}
-; CHECK-LABEL: @all_to_cast
-; CHECK: load <4 x float>
define void @all_to_cast(ptr nocapture readonly align 16 dereferenceable(16) %alloc1) {
+; CHECK-LABEL: define void @all_to_cast(
+; CHECK-SAME: ptr readonly align 16 captures(none) dereferenceable(16) [[ALLOC1:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[ALLOC16:%.*]] = addrspacecast ptr [[ALLOC1]] to ptr addrspace(1)
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b32>, ptr addrspace(1) [[ALLOC16]], align 16, !invariant.load [[META0:![0-9]+]]
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <4 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[TMP82]] to float
+; CHECK-NEXT: [[TMP173:%.*]] = extractelement <4 x b32> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[TMP173]] to float
+; CHECK-NEXT: [[TMP264:%.*]] = extractelement <4 x b32> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP264]] to float
+; CHECK-NEXT: ret void
+;
entry:
%alloc16 = addrspacecast ptr %alloc1 to ptr addrspace(1)
%tmp1 = load float, ptr addrspace(1) %alloc16, align 16, !invariant.load !0
@@ -51,9 +78,18 @@ entry:
ret void
}
-; CHECK-LABEL: @ext_ptr
-; CHECK: load <2 x i32>
define void @ext_ptr(ptr addrspace(5) %p) {
+; CHECK-LABEL: define void @ext_ptr(
+; CHECK-SAME: ptr addrspace(5) [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[P]] to ptr
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr [[A_ASCAST]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[TMP11]] to i32
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[TMP22]] to i32
+; CHECK-NEXT: unreachable
+;
entry:
%gep2 = getelementptr inbounds i32, ptr addrspace(5) %p, i64 1
%a.ascast = addrspacecast ptr addrspace(5) %p to ptr
@@ -63,9 +99,20 @@ entry:
unreachable
}
-; CHECK-LABEL: @select_different_as
-; CHECK: load <2 x i32>
define void @select_different_as(ptr addrspace(1) %p0, ptr addrspace(5) %q0, i1 %cond) {
+; CHECK-LABEL: define void @select_different_as(
+; CHECK-SAME: ptr addrspace(1) [[P0:%.*]], ptr addrspace(5) [[Q0:%.*]], i1 [[COND:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P0_ASCAST:%.*]] = addrspacecast ptr addrspace(1) [[P0]] to ptr
+; CHECK-NEXT: [[Q0_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[Q0]] to ptr
+; CHECK-NEXT: [[SEL0:%.*]] = select i1 [[COND]], ptr [[P0_ASCAST]], ptr [[Q0_ASCAST]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr [[SEL0]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[TMP11]] to i32
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[TMP22]] to i32
+; CHECK-NEXT: unreachable
+;
entry:
%p1 = getelementptr inbounds i32, ptr addrspace(1) %p0, i64 1
%q1 = getelementptr inbounds i32, ptr addrspace(5) %q0, i64 1
@@ -80,9 +127,18 @@ entry:
unreachable
}
-; CHECK-LABEL: @shrink_ptr
-; CHECK: load <2 x i32>
define void @shrink_ptr(ptr %p) {
+; CHECK-LABEL: define void @shrink_ptr(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A_ASCAST:%.*]] = addrspacecast ptr [[P]] to ptr addrspace(5)
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr addrspace(5) [[A_ASCAST]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[TMP11]] to i32
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[TMP22]] to i32
+; CHECK-NEXT: unreachable
+;
entry:
%gep2 = getelementptr inbounds i32, ptr %p, i64 1
%a.ascast = addrspacecast ptr %p to ptr addrspace(5)
@@ -92,9 +148,19 @@ entry:
unreachable
}
-; CHECK-LABEL: @ext_ptr_wrap
-; CHECK: load <2 x i8>
define void @ext_ptr_wrap(ptr addrspace(5) %p) {
+; CHECK-LABEL: define void @ext_ptr_wrap(
+; CHECK-SAME: ptr addrspace(5) [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i8, ptr addrspace(5) [[P]], i64 4294967295
+; CHECK-NEXT: [[B_ASCAST:%.*]] = addrspacecast ptr addrspace(5) [[GEP2]] to ptr
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b8>, ptr [[B_ASCAST]], align 2
+; CHECK-NEXT: [[TMP21:%.*]] = extractelement <2 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b8 [[TMP21]] to i8
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[TMP12]] to i8
+; CHECK-NEXT: unreachable
+;
entry:
%gep2 = getelementptr inbounds i8, ptr addrspace(5) %p, i64 4294967295
%a.ascast = addrspacecast ptr addrspace(5) %p to ptr
@@ -105,3 +171,6 @@ entry:
}
!0 = !{}
+;.
+; CHECK: [[META0]] = !{}
+;.
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vectorize-redund-loads.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vectorize-redund-loads.ll
index 802795da47894..e350ce894d262 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vectorize-redund-loads.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/vectorize-redund-loads.ll
@@ -4,15 +4,17 @@
define void @onevec(ptr %ptr) {
; CHECK-LABEL: define void @onevec(
; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, ptr [[PTR]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = bitcast i32 [[TMP1]] to <1 x i32>
+; CHECK-NEXT: [[TMP1:%.*]] = load b32, ptr [[PTR]], align 4
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[TMP1]] to <1 x i32>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[TMP1]] to i32
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 16
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, ptr [[GEP1]], align 4
-; CHECK-NEXT: [[TMP4:%.*]] = bitcast i32 [[TMP3]] to <1 x i32>
+; CHECK-NEXT: [[TMP4:%.*]] = load b32, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[TMP4]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[TMP4]] to <1 x i32>
; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 32
-; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[GEP2]], align 4
-; CHECK-NEXT: [[TMP6:%.*]] = bitcast i32 [[TMP5]] to <1 x i32>
-; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32 [[TMP5]] to <1 x i32>
+; CHECK-NEXT: [[TMP7:%.*]] = load b32, ptr [[GEP2]], align 4
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b32 [[TMP7]] to <1 x i32>
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast b32 [[TMP7]] to <1 x i32>
; CHECK-NEXT: ret void
;
%ld0 = load <1 x i32>, ptr %ptr, align 4
@@ -31,11 +33,15 @@ define void @onevec(ptr %ptr) {
define void @test(ptr %ptr) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[PTR]], align 4
-; CHECK-NEXT: [[LD01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LD12:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <2 x i32> <i32 1, i32 2>
-; CHECK-NEXT: [[LD23:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[LD34:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr [[PTR]], align 4
+; CHECK-NEXT: [[LD01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD01]] to i32
+; CHECK-NEXT: [[LD12:%.*]] = shufflevector <4 x b32> [[TMP1]], <4 x b32> poison, <2 x i32> <i32 1, i32 2>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b32> [[LD12]] to <2 x i32>
+; CHECK-NEXT: [[LD23:%.*]] = shufflevector <4 x b32> [[TMP1]], <4 x b32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x b32> [[LD23]] to <2 x i32>
+; CHECK-NEXT: [[LD34:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LD34]] to i32
; CHECK-NEXT: ret void
;
%ld0 = load i32, ptr %ptr, align 4
@@ -52,15 +58,23 @@ define void @test(ptr %ptr) {
define void @test2() {
; CHECK-LABEL: define void @test2() {
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr addrspace(1) @ptr, align 4
-; CHECK-NEXT: [[VECINS1:%.*]] = shufflevector <8 x float> [[TMP1]], <8 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP3:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP5:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
-; CHECK-NEXT: [[TMP7:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x b32>, ptr addrspace(1) @ptr, align 4
+; CHECK-NEXT: [[VECINS2:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[VECINS1:%.*]] = bitcast <8 x b32> [[VECINS2]] to <8 x float>
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <8 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[TMP10]] to float
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <8 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[TMP12]] to float
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <8 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP14]] to float
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <8 x b32> [[TMP1]], i32 4
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[TMP9]] to float
+; CHECK-NEXT: [[TMP11:%.*]] = extractelement <8 x b32> [[TMP1]], i32 5
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[TMP11]] to float
+; CHECK-NEXT: [[TMP13:%.*]] = extractelement <8 x b32> [[TMP1]], i32 6
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b32 [[TMP13]] to float
+; CHECK-NEXT: [[TMP15:%.*]] = extractelement <8 x b32> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b32 [[TMP15]] to float
; CHECK-NEXT: [[VECINS_1:%.*]] = insertelement <8 x float> [[VECINS1]], float [[TMP2]], i64 1
; CHECK-NEXT: [[VECINS_2:%.*]] = insertelement <8 x float> [[VECINS_1]], float [[TMP3]], i64 2
; CHECK-NEXT: [[VECINS_3:%.*]] = insertelement <8 x float> [[VECINS_2]], float [[TMP4]], i64 3
@@ -94,9 +108,11 @@ define void @vect_zext_bitcast_i8_st4_to_i32_idx(ptr addrspace(1) %arg1, i32 %ba
; CHECK-NEXT: [[ADD1:%.*]] = add nuw i32 [[BASE]], 0
; CHECK-NEXT: [[ZEXT1:%.*]] = zext i32 [[ADD1]] to i64
; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ARG1]], i64 [[ZEXT1]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr addrspace(1) [[GEP1]], align 4
-; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr addrspace(1) [[GEP1]], align 4
+; CHECK-NEXT: [[LOAD11:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LOAD11]] to i32
+; CHECK-NEXT: [[LOAD22:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LOAD22]] to i32
; CHECK-NEXT: [[ADD25:%.*]] = add nuw i32 [[BASE]], 6
; CHECK-NEXT: [[ZEXT25:%.*]] = zext i32 [[ADD25]] to i64
; CHECK-NEXT: [[GEP25:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ARG1]], i64 [[ZEXT25]]
@@ -104,9 +120,11 @@ define void @vect_zext_bitcast_i8_st4_to_i32_idx(ptr addrspace(1) %arg1, i32 %ba
; CHECK-NEXT: [[ADD3:%.*]] = add nuw i32 [[BASE]], 8
; CHECK-NEXT: [[ZEXT3:%.*]] = zext i32 [[ADD3]] to i64
; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[ARG1]], i64 [[ZEXT3]]
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr addrspace(1) [[GEP3]], align 4
-; CHECK-NEXT: [[LOAD33:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
-; CHECK-NEXT: [[LOAD44:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x b32>, ptr addrspace(1) [[GEP3]], align 4
+; CHECK-NEXT: [[LOAD33:%.*]] = extractelement <2 x b32> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LOAD33]] to i32
+; CHECK-NEXT: [[LOAD44:%.*]] = extractelement <2 x b32> [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[LOAD44]] to i32
; CHECK-NEXT: ret void
;
%add1 = add nuw i32 %base, 0
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll
index 842268a8b853b..3d1aa961f6aa0 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/weird-type-accesses.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=amdgcn-amd-amdhsa -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; Checks that we don't merge loads/stores of types smaller than one
@@ -13,50 +14,70 @@ declare void @use_v2i2(<2 x i2>)
declare void @use_v4i2(<4 x i2>)
declare void @use_v2i9(<2 x i9>)
-; CHECK-LABEL: @merge_store_2_constants_i1(
-; CHECK: store i1
-; CHECK: store i1
define amdgpu_kernel void @merge_store_2_constants_i1(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_2_constants_i1(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i1, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i1 true, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: store i1 false, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i1, ptr addrspace(1) %out, i32 1
store i1 true, ptr addrspace(1) %out.gep.1
store i1 false, ptr addrspace(1) %out
ret void
}
-; CHECK-LABEL: @merge_store_2_constants_i2(
-; CHECK: store i2 1
-; CHECK: store i2 -1
define amdgpu_kernel void @merge_store_2_constants_i2(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_2_constants_i2(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i2, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i2 1, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: store i2 -1, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i2, ptr addrspace(1) %out, i32 1
store i2 1, ptr addrspace(1) %out.gep.1
store i2 -1, ptr addrspace(1) %out
ret void
}
-; CHECK-LABEL: @merge_different_store_sizes_i1_i8(
-; CHECK: store i1 true
-; CHECK: store i8 123
define amdgpu_kernel void @merge_different_store_sizes_i1_i8(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_different_store_sizes_i1_i8(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i8, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i1 true, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: store i8 123, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i8, ptr addrspace(1) %out, i32 1
store i1 true, ptr addrspace(1) %out
store i8 123, ptr addrspace(1) %out.gep.1
ret void
}
-; CHECK-LABEL: @merge_different_store_sizes_i8_i1(
-; CHECK: store i8 123
-; CHECK: store i1 true
define amdgpu_kernel void @merge_different_store_sizes_i8_i1(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_different_store_sizes_i8_i1(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i8, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i8 123, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: store i1 true, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i8, ptr addrspace(1) %out, i32 1
store i8 123, ptr addrspace(1) %out.gep.1
store i1 true, ptr addrspace(1) %out
ret void
}
-; CHECK-LABEL: @merge_store_2_constant_structs(
-; CHECK: store %struct.foo
-; CHECK: store %struct.foo
define amdgpu_kernel void @merge_store_2_constant_structs(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_2_constant_structs(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store [[STRUCT_FOO]] { i32 12, i8 3 }, ptr addrspace(1) [[OUT_GEP_1]], align 4
+; CHECK-NEXT: store [[STRUCT_FOO]] { i32 92, i8 9 }, ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr %struct.foo, ptr addrspace(1) %out, i32 1
store %struct.foo { i32 12, i8 3 }, ptr addrspace(1) %out.gep.1
store %struct.foo { i32 92, i8 9 }, ptr addrspace(1) %out
@@ -64,10 +85,14 @@ define amdgpu_kernel void @merge_store_2_constant_structs(ptr addrspace(1) %out)
}
; sub-byte element size
-; CHECK-LABEL: @merge_store_2_constants_v2i2(
-; CHECK: store <2 x i2>
-; CHECK: store <2 x i2>
define amdgpu_kernel void @merge_store_2_constants_v2i2(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_2_constants_v2i2(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr <2 x i2>, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store <2 x i2> <i2 1, i2 -1>, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: store <2 x i2> <i2 -1, i2 1>, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr <2 x i2>, ptr addrspace(1) %out, i32 1
store <2 x i2> <i2 1, i2 -1>, ptr addrspace(1) %out.gep.1
store <2 x i2> <i2 -1, i2 1>, ptr addrspace(1) %out
@@ -76,20 +101,30 @@ define amdgpu_kernel void @merge_store_2_constants_v2i2(ptr addrspace(1) %out) #
; sub-byte element size but byte size
-; CHECK-LABEL: @merge_store_2_constants_v4i2(
-; CHECK: store <4 x i2>
-; CHECK: store <4 x i2>
define amdgpu_kernel void @merge_store_2_constants_v4i2(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_2_constants_v4i2(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr <4 x i2>, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store <4 x i2> <i2 1, i2 -1, i2 1, i2 -1>, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: store <4 x i2> <i2 -1, i2 1, i2 -1, i2 1>, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr <4 x i2>, ptr addrspace(1) %out, i32 1
store <4 x i2> <i2 1, i2 -1, i2 1, i2 -1>, ptr addrspace(1) %out.gep.1
store <4 x i2> <i2 -1, i2 1, i2 -1, i2 1>, ptr addrspace(1) %out
ret void
}
-; CHECK-LABEL: @merge_load_2_constants_i1(
-; CHECK: load i1
-; CHECK: load i1
define amdgpu_kernel void @merge_load_2_constants_i1(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_2_constants_i1(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i1, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: [[X:%.*]] = load i1, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: [[Y:%.*]] = load i1, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: call void @use_i1(i1 [[X]])
+; CHECK-NEXT: call void @use_i1(i1 [[Y]])
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i1, ptr addrspace(1) %out, i32 1
%x = load i1, ptr addrspace(1) %out.gep.1
%y = load i1, ptr addrspace(1) %out
@@ -98,10 +133,16 @@ define amdgpu_kernel void @merge_load_2_constants_i1(ptr addrspace(1) %out) #0 {
ret void
}
-; CHECK-LABEL: @merge_load_2_constants_i2(
-; CHECK: load i2
-; CHECK: load i2
define amdgpu_kernel void @merge_load_2_constants_i2(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_2_constants_i2(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i2, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: [[X:%.*]] = load i2, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: [[Y:%.*]] = load i2, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: call void @use_i2(i2 [[X]])
+; CHECK-NEXT: call void @use_i2(i2 [[Y]])
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i2, ptr addrspace(1) %out, i32 1
%x = load i2, ptr addrspace(1) %out.gep.1
%y = load i2, ptr addrspace(1) %out
@@ -110,10 +151,16 @@ define amdgpu_kernel void @merge_load_2_constants_i2(ptr addrspace(1) %out) #0 {
ret void
}
-; CHECK-LABEL: @merge_different_load_sizes_i1_i8(
-; CHECK: load i1
-; CHECK: load i8
define amdgpu_kernel void @merge_different_load_sizes_i1_i8(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_different_load_sizes_i1_i8(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i8, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: [[X:%.*]] = load i1, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: [[Y:%.*]] = load i8, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: call void @use_i1(i1 [[X]])
+; CHECK-NEXT: call void @use_i8(i8 [[Y]])
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i8, ptr addrspace(1) %out, i32 1
%x = load i1, ptr addrspace(1) %out
%y = load i8, ptr addrspace(1) %out.gep.1
@@ -122,10 +169,16 @@ define amdgpu_kernel void @merge_different_load_sizes_i1_i8(ptr addrspace(1) %ou
ret void
}
-; CHECK-LABEL: @merge_different_load_sizes_i8_i1(
-; CHECK: load i8
-; CHECK: load i1
define amdgpu_kernel void @merge_different_load_sizes_i8_i1(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_different_load_sizes_i8_i1(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i8, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: [[X:%.*]] = load i8, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: [[Y:%.*]] = load i1, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: call void @use_i8(i8 [[X]])
+; CHECK-NEXT: call void @use_i1(i1 [[Y]])
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i8, ptr addrspace(1) %out, i32 1
%x = load i8, ptr addrspace(1) %out.gep.1
%y = load i1, ptr addrspace(1) %out
@@ -134,10 +187,16 @@ define amdgpu_kernel void @merge_different_load_sizes_i8_i1(ptr addrspace(1) %ou
ret void
}
-; CHECK-LABEL: @merge_load_2_constant_structs(
-; CHECK: load %struct.foo
-; CHECK: load %struct.foo
define amdgpu_kernel void @merge_load_2_constant_structs(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_2_constant_structs(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr [[STRUCT_FOO:%.*]], ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: [[X:%.*]] = load [[STRUCT_FOO]], ptr addrspace(1) [[OUT_GEP_1]], align 4
+; CHECK-NEXT: [[Y:%.*]] = load [[STRUCT_FOO]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: call void @use_foo([[STRUCT_FOO]] [[X]])
+; CHECK-NEXT: call void @use_foo([[STRUCT_FOO]] [[Y]])
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr %struct.foo, ptr addrspace(1) %out, i32 1
%x = load %struct.foo, ptr addrspace(1) %out.gep.1
%y = load %struct.foo, ptr addrspace(1) %out
@@ -146,10 +205,16 @@ define amdgpu_kernel void @merge_load_2_constant_structs(ptr addrspace(1) %out)
ret void
}
-; CHECK-LABEL: @merge_load_2_constants_v2i2(
-; CHECK: load <2 x i2>
-; CHECK: load <2 x i2>
define amdgpu_kernel void @merge_load_2_constants_v2i2(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_2_constants_v2i2(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr <2 x i2>, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: [[X:%.*]] = load <2 x i2>, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: [[Y:%.*]] = load <2 x i2>, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: call void @use_v2i2(<2 x i2> [[X]])
+; CHECK-NEXT: call void @use_v2i2(<2 x i2> [[Y]])
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr <2 x i2>, ptr addrspace(1) %out, i32 1
%x = load <2 x i2>, ptr addrspace(1) %out.gep.1
%y = load <2 x i2>, ptr addrspace(1) %out
@@ -158,10 +223,16 @@ define amdgpu_kernel void @merge_load_2_constants_v2i2(ptr addrspace(1) %out) #0
ret void
}
-; CHECK-LABEL: @merge_load_2_constants_v4i2(
-; CHECK: load <4 x i2>
-; CHECK: load <4 x i2>
define amdgpu_kernel void @merge_load_2_constants_v4i2(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_2_constants_v4i2(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr <4 x i2>, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: [[X:%.*]] = load <4 x i2>, ptr addrspace(1) [[OUT_GEP_1]], align 1
+; CHECK-NEXT: [[Y:%.*]] = load <4 x i2>, ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: call void @use_v4i2(<4 x i2> [[X]])
+; CHECK-NEXT: call void @use_v4i2(<4 x i2> [[Y]])
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr <4 x i2>, ptr addrspace(1) %out, i32 1
%x = load <4 x i2>, ptr addrspace(1) %out.gep.1
%y = load <4 x i2>, ptr addrspace(1) %out
@@ -170,20 +241,30 @@ define amdgpu_kernel void @merge_load_2_constants_v4i2(ptr addrspace(1) %out) #0
ret void
}
-; CHECK-LABEL: @merge_store_2_constants_i9(
-; CHECK: store i9 3
-; CHECK: store i9 -5
define amdgpu_kernel void @merge_store_2_constants_i9(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_store_2_constants_i9(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr i9, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i9 3, ptr addrspace(1) [[OUT_GEP_1]], align 2
+; CHECK-NEXT: store i9 -5, ptr addrspace(1) [[OUT]], align 2
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr i9, ptr addrspace(1) %out, i32 1
store i9 3, ptr addrspace(1) %out.gep.1
store i9 -5, ptr addrspace(1) %out
ret void
}
-; CHECK-LABEL: @merge_load_2_constants_v2i9(
-; CHECK: load <2 x i9>
-; CHECK: load <2 x i9>
define amdgpu_kernel void @merge_load_2_constants_v2i9(ptr addrspace(1) %out) #0 {
+; CHECK-LABEL: define amdgpu_kernel void @merge_load_2_constants_v2i9(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[OUT_GEP_1:%.*]] = getelementptr <2 x i9>, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: [[X:%.*]] = load <2 x i9>, ptr addrspace(1) [[OUT_GEP_1]], align 4
+; CHECK-NEXT: [[Y:%.*]] = load <2 x i9>, ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: call void @use_v2i9(<2 x i9> [[X]])
+; CHECK-NEXT: call void @use_v2i9(<2 x i9> [[Y]])
+; CHECK-NEXT: ret void
+;
%out.gep.1 = getelementptr <2 x i9>, ptr addrspace(1) %out, i32 1
%x = load <2 x i9>, ptr addrspace(1) %out.gep.1
%y = load <2 x i9>, ptr addrspace(1) %out
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/4x2xhalf.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/4x2xhalf.ll
index 89f2be8494f67..e1bdf7a25b08f 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/4x2xhalf.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/4x2xhalf.ll
@@ -4,11 +4,15 @@
define void @ldg_f16(ptr nocapture align 16 %rd0) {
; CHECK-LABEL: define void @ldg_f16(
; CHECK-SAME: ptr align 16 captures(none) [[RD0:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x half>, ptr [[RD0]], align 16
-; CHECK-NEXT: [[LOAD11:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[LOAD22:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[LOAD33:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: [[LOAD44:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x b16>, ptr [[RD0]], align 16
+; CHECK-NEXT: [[LOAD12:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[LOAD11:%.*]] = bitcast <2 x b16> [[LOAD12]] to <2 x half>
+; CHECK-NEXT: [[LOAD23:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[LOAD22:%.*]] = bitcast <2 x b16> [[LOAD23]] to <2 x half>
+; CHECK-NEXT: [[LOAD34:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: [[LOAD33:%.*]] = bitcast <2 x b16> [[LOAD34]] to <2 x half>
+; CHECK-NEXT: [[LOAD45:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[LOAD44:%.*]] = bitcast <2 x b16> [[LOAD45]] to <2 x half>
; CHECK-NEXT: [[P1:%.*]] = fcmp ogt <2 x half> [[LOAD11]], zeroinitializer
; CHECK-NEXT: [[S1:%.*]] = select <2 x i1> [[P1]], <2 x half> [[LOAD11]], <2 x half> zeroinitializer
; CHECK-NEXT: [[P2:%.*]] = fcmp ogt <2 x half> [[LOAD22]], zeroinitializer
@@ -17,23 +21,27 @@ define void @ldg_f16(ptr nocapture align 16 %rd0) {
; CHECK-NEXT: [[S3:%.*]] = select <2 x i1> [[P3]], <2 x half> [[LOAD33]], <2 x half> zeroinitializer
; CHECK-NEXT: [[P4:%.*]] = fcmp ogt <2 x half> [[LOAD44]], zeroinitializer
; CHECK-NEXT: [[S4:%.*]] = select <2 x i1> [[P4]], <2 x half> [[LOAD44]], <2 x half> zeroinitializer
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <2 x half> [[S1]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x half> poison, half [[TMP2]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <2 x half> [[S1]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x half> [[TMP3]], half [[TMP4]], i32 1
-; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x half> [[S2]], i32 0
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x half> [[TMP5]], half [[TMP6]], i32 2
-; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x half> [[S2]], i32 1
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x half> [[TMP7]], half [[TMP8]], i32 3
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x half> [[S3]], i32 0
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x half> [[TMP9]], half [[TMP10]], i32 4
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x half> [[S3]], i32 1
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x half> [[TMP11]], half [[TMP12]], i32 5
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x half> [[S4]], i32 0
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x half> [[TMP13]], half [[TMP14]], i32 6
-; CHECK-NEXT: [[TMP16:%.*]] = extractelement <2 x half> [[S4]], i32 1
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x half> [[TMP15]], half [[TMP16]], i32 7
-; CHECK-NEXT: store <8 x half> [[TMP17]], ptr [[RD0]], align 16
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x half> [[S1]] to <2 x b16>
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x b16> [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x b16> poison, b16 [[TMP7]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x b16> [[TMP6]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x b16> [[TMP8]], b16 [[TMP9]], i32 1
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast <2 x half> [[S2]] to <2 x b16>
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x b16> [[TMP11]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x b16> [[TMP10]], b16 [[TMP12]], i32 2
+; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x b16> [[TMP11]], i32 1
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x b16> [[TMP13]], b16 [[TMP14]], i32 3
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast <2 x half> [[S3]] to <2 x b16>
+; CHECK-NEXT: [[TMP17:%.*]] = extractelement <2 x b16> [[TMP16]], i32 0
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <8 x b16> [[TMP15]], b16 [[TMP17]], i32 4
+; CHECK-NEXT: [[TMP19:%.*]] = extractelement <2 x b16> [[TMP16]], i32 1
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <8 x b16> [[TMP18]], b16 [[TMP19]], i32 5
+; CHECK-NEXT: [[TMP21:%.*]] = bitcast <2 x half> [[S4]] to <2 x b16>
+; CHECK-NEXT: [[TMP22:%.*]] = extractelement <2 x b16> [[TMP21]], i32 0
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <8 x b16> [[TMP20]], b16 [[TMP22]], i32 6
+; CHECK-NEXT: [[TMP24:%.*]] = extractelement <2 x b16> [[TMP21]], i32 1
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <8 x b16> [[TMP23]], b16 [[TMP24]], i32 7
+; CHECK-NEXT: store <8 x b16> [[TMP25]], ptr [[RD0]], align 16
; CHECK-NEXT: ret void
;
%load1 = load <2 x half>, ptr %rd0, align 16
@@ -62,11 +70,13 @@ define void @ldg_f16(ptr nocapture align 16 %rd0) {
define void @no_nonpow2_vector(ptr nocapture align 16 %rd0) {
; CHECK-LABEL: define void @no_nonpow2_vector(
; CHECK-SAME: ptr align 16 captures(none) [[RD0:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x half> @llvm.masked.load.v8f16.p0(ptr align 16 [[RD0]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false>, <8 x half> poison)
-; CHECK-NEXT: [[LOAD13:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <3 x i32> <i32 0, i32 1, i32 2>
-; CHECK-NEXT: [[LOAD24:%.*]] = shufflevector <8 x half> [[TMP1]], <8 x half> poison, <3 x i32> <i32 3, i32 4, i32 5>
-; CHECK-NEXT: [[EXTEND5:%.*]] = extractelement <8 x half> [[TMP1]], i32 6
-; CHECK-NEXT: [[EXTEND26:%.*]] = extractelement <8 x half> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x b16> @llvm.masked.load.v8b16.p0(ptr align 16 [[RD0]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false>, <8 x b16> poison)
+; CHECK-NEXT: [[LOAD14:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: [[LOAD13:%.*]] = bitcast <3 x b16> [[LOAD14]] to <3 x half>
+; CHECK-NEXT: [[LOAD25:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <3 x i32> <i32 3, i32 4, i32 5>
+; CHECK-NEXT: [[LOAD24:%.*]] = bitcast <3 x b16> [[LOAD25]] to <3 x half>
+; CHECK-NEXT: [[EXTEND5:%.*]] = extractelement <8 x b16> [[TMP1]], i32 6
+; CHECK-NEXT: [[EXTEND26:%.*]] = extractelement <8 x b16> [[TMP1]], i32 7
; CHECK-NEXT: [[P1:%.*]] = fcmp ogt <3 x half> [[LOAD13]], zeroinitializer
; CHECK-NEXT: [[S1:%.*]] = select <3 x i1> [[P1]], <3 x half> [[LOAD13]], <3 x half> zeroinitializer
; CHECK-NEXT: store <3 x half> [[S1]], ptr [[RD0]], align 16
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/extend-chain.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/extend-chain.ll
index 5c3757867f71f..be595763ae3dd 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/extend-chain.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/extend-chain.ll
@@ -10,11 +10,14 @@ define void @load3to4(ptr %p) {
; CHECK-LABEL: define void @load3to4(
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[P_0:%.*]] = getelementptr i32, ptr [[P]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[P_0]], <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x i32> poison)
-; CHECK-NEXT: [[V01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[V12:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[V23:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[EXTEND4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[P_0]], <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x b32> poison)
+; CHECK-NEXT: [[V01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[V01]] to i32
+; CHECK-NEXT: [[V12:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[V12]] to i32
+; CHECK-NEXT: [[V23:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[V23]] to i32
+; CHECK-NEXT: [[EXTEND4:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
; CHECK-NEXT: ret void
;
%p.0 = getelementptr i32, ptr %p, i32 0
@@ -32,15 +35,20 @@ define void @load5to8(ptr %p) {
; CHECK-LABEL: define void @load5to8(
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[P_0:%.*]] = getelementptr i16, ptr [[P]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 16 [[P_0]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>, <8 x i16> poison)
-; CHECK-NEXT: [[V05:%.*]] = extractelement <8 x i16> [[TMP1]], i32 0
-; CHECK-NEXT: [[V16:%.*]] = extractelement <8 x i16> [[TMP1]], i32 1
-; CHECK-NEXT: [[V27:%.*]] = extractelement <8 x i16> [[TMP1]], i32 2
-; CHECK-NEXT: [[V38:%.*]] = extractelement <8 x i16> [[TMP1]], i32 3
-; CHECK-NEXT: [[V49:%.*]] = extractelement <8 x i16> [[TMP1]], i32 4
-; CHECK-NEXT: [[EXTEND10:%.*]] = extractelement <8 x i16> [[TMP1]], i32 5
-; CHECK-NEXT: [[EXTEND211:%.*]] = extractelement <8 x i16> [[TMP1]], i32 6
-; CHECK-NEXT: [[EXTEND412:%.*]] = extractelement <8 x i16> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x b16> @llvm.masked.load.v8b16.p0(ptr align 16 [[P_0]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>, <8 x b16> poison)
+; CHECK-NEXT: [[V05:%.*]] = extractelement <8 x b16> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b16 [[V05]] to i16
+; CHECK-NEXT: [[V16:%.*]] = extractelement <8 x b16> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b16 [[V16]] to i16
+; CHECK-NEXT: [[V27:%.*]] = extractelement <8 x b16> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b16 [[V27]] to i16
+; CHECK-NEXT: [[V38:%.*]] = extractelement <8 x b16> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b16 [[V38]] to i16
+; CHECK-NEXT: [[V49:%.*]] = extractelement <8 x b16> [[TMP1]], i32 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b16 [[V49]] to i16
+; CHECK-NEXT: [[EXTEND10:%.*]] = extractelement <8 x b16> [[TMP1]], i32 5
+; CHECK-NEXT: [[EXTEND211:%.*]] = extractelement <8 x b16> [[TMP1]], i32 6
+; CHECK-NEXT: [[EXTEND412:%.*]] = extractelement <8 x b16> [[TMP1]], i32 7
; CHECK-NEXT: ret void
;
%p.0 = getelementptr i16, ptr %p, i32 0
@@ -62,15 +70,21 @@ define void @load6to8(ptr %p) {
; CHECK-LABEL: define void @load6to8(
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[P_0:%.*]] = getelementptr i16, ptr [[P]], i32 0
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 16 [[P_0]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false>, <8 x i16> poison)
-; CHECK-NEXT: [[V05:%.*]] = extractelement <8 x i16> [[TMP1]], i32 0
-; CHECK-NEXT: [[V16:%.*]] = extractelement <8 x i16> [[TMP1]], i32 1
-; CHECK-NEXT: [[V27:%.*]] = extractelement <8 x i16> [[TMP1]], i32 2
-; CHECK-NEXT: [[V38:%.*]] = extractelement <8 x i16> [[TMP1]], i32 3
-; CHECK-NEXT: [[V49:%.*]] = extractelement <8 x i16> [[TMP1]], i32 4
-; CHECK-NEXT: [[EXTEND10:%.*]] = extractelement <8 x i16> [[TMP1]], i32 5
-; CHECK-NEXT: [[EXTEND211:%.*]] = extractelement <8 x i16> [[TMP1]], i32 6
-; CHECK-NEXT: [[EXTEND412:%.*]] = extractelement <8 x i16> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x b16> @llvm.masked.load.v8b16.p0(ptr align 16 [[P_0]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false>, <8 x b16> poison)
+; CHECK-NEXT: [[V03:%.*]] = extractelement <8 x b16> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b16 [[V03]] to i16
+; CHECK-NEXT: [[V14:%.*]] = extractelement <8 x b16> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b16 [[V14]] to i16
+; CHECK-NEXT: [[V25:%.*]] = extractelement <8 x b16> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b16 [[V25]] to i16
+; CHECK-NEXT: [[V36:%.*]] = extractelement <8 x b16> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b16 [[V36]] to i16
+; CHECK-NEXT: [[V47:%.*]] = extractelement <8 x b16> [[TMP1]], i32 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b16 [[V47]] to i16
+; CHECK-NEXT: [[V58:%.*]] = extractelement <8 x b16> [[TMP1]], i32 5
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b16 [[V58]] to i16
+; CHECK-NEXT: [[EXTEND9:%.*]] = extractelement <8 x b16> [[TMP1]], i32 6
+; CHECK-NEXT: [[EXTEND210:%.*]] = extractelement <8 x b16> [[TMP1]], i32 7
; CHECK-NEXT: ret void
;
%p.0 = getelementptr i16, ptr %p, i32 0
@@ -95,9 +109,11 @@ define void @load3to4_unaligned(ptr %p) {
; CHECK-SAME: ptr [[P:%.*]]) {
; CHECK-NEXT: [[P_0:%.*]] = getelementptr i32, ptr [[P]], i32 0
; CHECK-NEXT: [[P_2:%.*]] = getelementptr i32, ptr [[P]], i32 2
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[P_0]], align 8
-; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[P_0]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[V01]] to i32
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[V12]] to i32
; CHECK-NEXT: [[V2:%.*]] = load i32, ptr [[P_2]], align 8
; CHECK-NEXT: ret void
;
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-cleanup.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-cleanup.ll
index 7b659b0feeb03..10711753309fc 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-cleanup.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-cleanup.ll
@@ -8,11 +8,15 @@
define void @fillTwoGapsCanVectorize(ptr %in) {
; CHECK-LABEL: define void @fillTwoGapsCanVectorize(
; CHECK-SAME: ptr [[IN:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[IN]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> poison)
-; CHECK-NEXT: [[LOAD03:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[GAPFILL4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[GAPFILL25:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[LOAD36:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[IN]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x b32> poison)
+; CHECK-NEXT: [[LOAD03:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LOAD03]] to i32
+; CHECK-NEXT: [[GAPFILL4:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[GAPFILL4]] to i32
+; CHECK-NEXT: [[GAPFILL25:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[GAPFILL25]] to i32
+; CHECK-NEXT: [[LOAD36:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LOAD36]] to i32
; CHECK-NEXT: ret void
;
%load0 = load i32, ptr %in, align 16
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-invariant.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-invariant.ll
index 145512863f4d7..8e47a185b28a0 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-invariant.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-invariant.ll
@@ -8,11 +8,15 @@
define i32 @noGaps(ptr %in) {
; CHECK-LABEL: define i32 @noGaps(
; CHECK-SAME: ptr [[IN:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[IN]], align 16, !invariant.load [[META0:![0-9]+]]
-; CHECK-NEXT: [[TMP01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr [[IN]], align 16, !invariant.load [[META0:![0-9]+]]
+; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP01:%.*]] = bitcast b32 [[LOAD01]] to i32
+; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast b32 [[LOAD12]] to i32
+; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP23:%.*]] = bitcast b32 [[LOAD23]] to i32
+; CHECK-NEXT: [[LOAD34:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP34:%.*]] = bitcast b32 [[LOAD34]] to i32
; CHECK-NEXT: [[SUM01:%.*]] = add i32 [[TMP01]], [[TMP12]]
; CHECK-NEXT: [[SUM012:%.*]] = add i32 [[SUM01]], [[TMP23]]
; CHECK-NEXT: [[SUM0123:%.*]] = add i32 [[SUM012]], [[TMP34]]
@@ -35,11 +39,15 @@ define i32 @noGaps(ptr %in) {
define i32 @noGapsMissingInvariant(ptr %in) {
; CHECK-LABEL: define i32 @noGapsMissingInvariant(
; CHECK-SAME: ptr [[IN:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[IN]], align 16
-; CHECK-NEXT: [[TMP01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP23:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[TMP34:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr [[IN]], align 16
+; CHECK-NEXT: [[LOAD01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP01:%.*]] = bitcast b32 [[LOAD01]] to i32
+; CHECK-NEXT: [[LOAD12:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast b32 [[LOAD12]] to i32
+; CHECK-NEXT: [[LOAD23:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP23:%.*]] = bitcast b32 [[LOAD23]] to i32
+; CHECK-NEXT: [[LOAD34:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP34:%.*]] = bitcast b32 [[LOAD34]] to i32
; CHECK-NEXT: [[SUM01:%.*]] = add i32 [[TMP01]], [[TMP12]]
; CHECK-NEXT: [[SUM012:%.*]] = add i32 [[SUM01]], [[TMP23]]
; CHECK-NEXT: [[SUM0123:%.*]] = add i32 [[SUM012]], [[TMP34]]
@@ -62,11 +70,15 @@ define i32 @noGapsMissingInvariant(ptr %in) {
define i32 @twoGaps(ptr %in) {
; CHECK-LABEL: define i32 @twoGaps(
; CHECK-SAME: ptr [[IN:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[IN]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> poison), !invariant.load [[META0]]
-; CHECK-NEXT: [[LOAD03:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[GAPFILL4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[GAPFILL25:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[LOAD36:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[IN]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x b32> poison), !invariant.load [[META0]]
+; CHECK-NEXT: [[LOAD3:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[LOAD03:%.*]] = bitcast b32 [[LOAD3]] to i32
+; CHECK-NEXT: [[GAPFILL4:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[GAPFILL4]] to i32
+; CHECK-NEXT: [[GAPFILL25:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[GAPFILL25]] to i32
+; CHECK-NEXT: [[LOAD37:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[LOAD36:%.*]] = bitcast b32 [[LOAD37]] to i32
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[LOAD03]], [[LOAD36]]
; CHECK-NEXT: ret i32 [[SUM]]
;
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-vectors.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-vectors.ll
index 15a92fbb452ac..1dfd8bcbed38a 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-vectors.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-vectors.ll
@@ -10,11 +10,15 @@ define void @i8x2_gap_gap_i8x2(ptr %ptr) {
; CHECK-LABEL: define void @i8x2_gap_gap_i8x2(
; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i8> @llvm.masked.load.v8i8.p0(ptr align 8 [[PTR0]], <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true>, <8 x i8> poison)
-; CHECK-NEXT: [[L03:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[GAPFILL4:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[GAPFILL25:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: [[L36:%.*]] = shufflevector <8 x i8> [[TMP1]], <8 x i8> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x b8> @llvm.masked.load.v8b8.p0(ptr align 8 [[PTR0]], <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true>, <8 x b8> poison)
+; CHECK-NEXT: [[L03:%.*]] = shufflevector <8 x b8> [[TMP1]], <8 x b8> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b8> [[L03]] to <2 x i8>
+; CHECK-NEXT: [[GAPFILL4:%.*]] = shufflevector <8 x b8> [[TMP1]], <8 x b8> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b8> [[GAPFILL4]] to <2 x i8>
+; CHECK-NEXT: [[GAPFILL25:%.*]] = shufflevector <8 x b8> [[TMP1]], <8 x b8> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x b8> [[GAPFILL25]] to <2 x i8>
+; CHECK-NEXT: [[L36:%.*]] = shufflevector <8 x b8> [[TMP1]], <8 x b8> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x b8> [[L36]] to <2 x i8>
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i64 0
@@ -50,11 +54,15 @@ define void @i16x2_gap_i16x2_i16x2(ptr %ptr) {
; CHECK-LABEL: define void @i16x2_gap_i16x2_i16x2(
; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 16 [[PTR0]], <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <8 x i16> poison)
-; CHECK-NEXT: [[L01:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[GAPFILL2:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[L23:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: [[L34:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x b16> @llvm.masked.load.v8b16.p0(ptr align 16 [[PTR0]], <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <8 x b16> poison)
+; CHECK-NEXT: [[L01:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b16> [[L01]] to <2 x i16>
+; CHECK-NEXT: [[GAPFILL2:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b16> [[GAPFILL2]] to <2 x i16>
+; CHECK-NEXT: [[L23:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x b16> [[L23]] to <2 x i16>
+; CHECK-NEXT: [[L34:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x b16> [[L34]] to <2 x i16>
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i64 0
@@ -72,11 +80,15 @@ define void @i16x2_gap_gap_i16x2(ptr %ptr) {
; CHECK-LABEL: define void @i16x2_gap_gap_i16x2(
; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i16> @llvm.masked.load.v8i16.p0(ptr align 16 [[PTR0]], <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true>, <8 x i16> poison)
-; CHECK-NEXT: [[L03:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[GAPFILL4:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[GAPFILL25:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: [[L36:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x b16> @llvm.masked.load.v8b16.p0(ptr align 16 [[PTR0]], <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 false, i1 false, i1 true, i1 true>, <8 x b16> poison)
+; CHECK-NEXT: [[L03:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b16> [[L03]] to <2 x i16>
+; CHECK-NEXT: [[GAPFILL4:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b16> [[GAPFILL4]] to <2 x i16>
+; CHECK-NEXT: [[GAPFILL25:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x b16> [[GAPFILL25]] to <2 x i16>
+; CHECK-NEXT: [[L36:%.*]] = shufflevector <8 x b16> [[TMP1]], <8 x b16> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x b16> [[L36]] to <2 x i16>
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i64 0
@@ -91,11 +103,15 @@ define void @i16x2_gap_gap_i16x2(ptr %ptr) {
define void @i32x2_i32x2_gap_i32x2(ptr addrspace(1) %in) {
; CHECK-LABEL: define void @i32x2_i32x2_gap_i32x2(
; CHECK-SAME: ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1(ptr addrspace(1) align 32 [[IN]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true>, <8 x i32> poison)
-; CHECK-NEXT: [[VEC01:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[VEC12:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[GAPFILL3:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: [[VEC34:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x b32> @llvm.masked.load.v8b32.p1(ptr addrspace(1) align 32 [[IN]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true>, <8 x b32> poison)
+; CHECK-NEXT: [[VEC01:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b32> [[VEC01]] to <2 x i32>
+; CHECK-NEXT: [[VEC12:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b32> [[VEC12]] to <2 x i32>
+; CHECK-NEXT: [[GAPFILL3:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x b32> [[GAPFILL3]] to <2 x i32>
+; CHECK-NEXT: [[VEC34:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x b32> [[VEC34]] to <2 x i32>
; CHECK-NEXT: ret void
;
%vec0 = load <2 x i32>, ptr addrspace(1) %in, align 32
@@ -113,9 +129,11 @@ define void @i64x2_gap_i64x2_i64x2(ptr addrspace(1) %in) {
; CHECK-SAME: ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[VEC0:%.*]] = load <2 x i64>, ptr addrspace(1) [[IN]], align 32
; CHECK-NEXT: [[GETELEM3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i32 32
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr addrspace(1) [[GETELEM3]], align 32
-; CHECK-NEXT: [[VEC31:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[VEC12:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b64>, ptr addrspace(1) [[GETELEM3]], align 32
+; CHECK-NEXT: [[VEC31:%.*]] = shufflevector <4 x b64> [[TMP1]], <4 x b64> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b64> [[VEC31]] to <2 x i64>
+; CHECK-NEXT: [[VEC12:%.*]] = shufflevector <4 x b64> [[TMP1]], <4 x b64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b64> [[VEC12]] to <2 x i64>
; CHECK-NEXT: ret void
;
%vec0 = load <2 x i64>, ptr addrspace(1) %in, align 32
@@ -131,9 +149,11 @@ define void @i64x2_gap_i64x2_i64x2(ptr addrspace(1) %in) {
define void @i64x2_i64x2_gap_i64x2(ptr addrspace(1) %in) {
; CHECK-LABEL: define void @i64x2_i64x2_gap_i64x2(
; CHECK-SAME: ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr addrspace(1) [[IN]], align 32
-; CHECK-NEXT: [[VEC01:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[VEC32:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b64>, ptr addrspace(1) [[IN]], align 32
+; CHECK-NEXT: [[VEC01:%.*]] = shufflevector <4 x b64> [[TMP1]], <4 x b64> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b64> [[VEC01]] to <2 x i64>
+; CHECK-NEXT: [[VEC32:%.*]] = shufflevector <4 x b64> [[TMP1]], <4 x b64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <2 x b64> [[VEC32]] to <2 x i64>
; CHECK-NEXT: [[GETELEM1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i32 48
; CHECK-NEXT: [[VEC1:%.*]] = load <2 x i64>, ptr addrspace(1) [[GETELEM1]], align 8
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-with-redundant-elements.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-with-redundant-elements.ll
index e22035b4ed922..6b29fad24e1a0 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-with-redundant-elements.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill-with-redundant-elements.ll
@@ -4,12 +4,17 @@
define void @test_redundant_no_gap(ptr addrspace(1) %ptr) {
; CHECK-LABEL: define void @test_redundant_no_gap(
; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i32>, ptr addrspace(1) [[PTR]], align 32
-; CHECK-NEXT: [[LD03:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[LD14:%.*]] = extractelement <8 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[LD25:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[LD37:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: [[LD45:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x b32>, ptr addrspace(1) [[PTR]], align 32
+; CHECK-NEXT: [[LD01:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b32> [[LD01]] to <2 x i32>
+; CHECK-NEXT: [[LD12:%.*]] = extractelement <8 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD12]] to i32
+; CHECK-NEXT: [[LD23:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x b32> [[LD23]] to <2 x i32>
+; CHECK-NEXT: [[LD34:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x b32> [[LD34]] to <2 x i32>
+; CHECK-NEXT: [[LD45:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x b32> [[LD45]] to <2 x i32>
; CHECK-NEXT: ret void
;
%ld0 = load <2 x i32>, ptr addrspace(1) %ptr, align 32
@@ -30,12 +35,17 @@ define void @test_redundant_no_gap(ptr addrspace(1) %ptr) {
define void @test_redundant_gap(ptr addrspace(1) %ptr) {
; CHECK-LABEL: define void @test_redundant_gap(
; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1(ptr addrspace(1) align 32 [[PTR]], <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <8 x i32> poison)
-; CHECK-NEXT: [[LD01:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[LD12:%.*]] = extractelement <8 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[GAPFILL3:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[LD34:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: [[LD45:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x b32> @llvm.masked.load.v8b32.p1(ptr addrspace(1) align 32 [[PTR]], <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 true, i1 true>, <8 x b32> poison)
+; CHECK-NEXT: [[LD01:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b32> [[LD01]] to <2 x i32>
+; CHECK-NEXT: [[LD12:%.*]] = extractelement <8 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD12]] to i32
+; CHECK-NEXT: [[GAPFILL3:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x b32> [[GAPFILL3]] to <2 x i32>
+; CHECK-NEXT: [[LD34:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x b32> [[LD34]] to <2 x i32>
+; CHECK-NEXT: [[LD45:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast <2 x b32> [[LD45]] to <2 x i32>
; CHECK-NEXT: ret void
;
%ld0 = load <2 x i32>, ptr addrspace(1) %ptr, align 32
@@ -52,14 +62,19 @@ define void @test_redundant_gap(ptr addrspace(1) %ptr) {
define void @test_redundant_gap_and_extend(ptr addrspace(1) %ptr) {
; CHECK-LABEL: define void @test_redundant_gap_and_extend(
; CHECK-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1(ptr addrspace(1) align 32 [[PTR]], <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>, <8 x i32> poison)
-; CHECK-NEXT: [[LD03:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[LD14:%.*]] = extractelement <8 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[GAPFILL5:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 2, i32 3>
-; CHECK-NEXT: [[LD36:%.*]] = shufflevector <8 x i32> [[TMP1]], <8 x i32> poison, <2 x i32> <i32 4, i32 5>
-; CHECK-NEXT: [[LD47:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4
-; CHECK-NEXT: [[EXTEND8:%.*]] = extractelement <8 x i32> [[TMP1]], i32 6
-; CHECK-NEXT: [[EXTEND29:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x b32> @llvm.masked.load.v8b32.p1(ptr addrspace(1) align 32 [[PTR]], <8 x i1> <i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false>, <8 x b32> poison)
+; CHECK-NEXT: [[LD03:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <2 x b32> [[LD03]] to <2 x i32>
+; CHECK-NEXT: [[LD14:%.*]] = extractelement <8 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD14]] to i32
+; CHECK-NEXT: [[GAPFILL5:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast <2 x b32> [[GAPFILL5]] to <2 x i32>
+; CHECK-NEXT: [[LD36:%.*]] = shufflevector <8 x b32> [[TMP1]], <8 x b32> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x b32> [[LD36]] to <2 x i32>
+; CHECK-NEXT: [[LD47:%.*]] = extractelement <8 x b32> [[TMP1]], i32 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[LD47]] to i32
+; CHECK-NEXT: [[EXTEND8:%.*]] = extractelement <8 x b32> [[TMP1]], i32 6
+; CHECK-NEXT: [[EXTEND29:%.*]] = extractelement <8 x b32> [[TMP1]], i32 7
; CHECK-NEXT: ret void
;
%ld0 = load <2 x i32>, ptr addrspace(1) %ptr, align 32
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill.ll
index 83152ece5c4d1..6939609a17e99 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/gap-fill.ll
@@ -5,11 +5,15 @@
define void @test(ptr %ptr) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
-; CHECK-NEXT: [[LD01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LD12:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[GAPFILL3:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[LD34:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x b32> poison)
+; CHECK-NEXT: [[LD01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD01]] to i32
+; CHECK-NEXT: [[LD12:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD12]] to i32
+; CHECK-NEXT: [[GAPFILL3:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[GAPFILL3]] to i32
+; CHECK-NEXT: [[LD34:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LD34]] to i32
; CHECK-NEXT: ret void
;
%ld0 = load i32, ptr %ptr, align 16
@@ -24,11 +28,15 @@ define void @test(ptr %ptr) {
define void @test2(ptr %ptr) {
; CHECK-LABEL: define void @test2(
; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x i32> poison)
-; CHECK-NEXT: [[LD01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[GAPFILL2:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[LD23:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[LD34:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x b32> poison)
+; CHECK-NEXT: [[LD01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD01]] to i32
+; CHECK-NEXT: [[GAPFILL2:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[GAPFILL2]] to i32
+; CHECK-NEXT: [[LD23:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[LD23]] to i32
+; CHECK-NEXT: [[LD34:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LD34]] to i32
; CHECK-NEXT: ret void
;
%ld0 = load i32, ptr %ptr, align 16
@@ -44,9 +52,11 @@ define void @test2(ptr %ptr) {
define void @test3(ptr %ptr) {
; CHECK-LABEL: define void @test3(
; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[PTR]], align 16
-; CHECK-NEXT: [[LD01:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT: [[LD12:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b64>, ptr [[PTR]], align 16
+; CHECK-NEXT: [[LD01:%.*]] = extractelement <2 x b64> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[LD01]] to i64
+; CHECK-NEXT: [[LD12:%.*]] = extractelement <2 x b64> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b64 [[LD12]] to i64
; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 24
; CHECK-NEXT: [[LD3:%.*]] = load i64, ptr [[GEP3]], align 4
; CHECK-NEXT: ret void
@@ -66,9 +76,11 @@ define void @test4(ptr %ptr) {
; CHECK-SAME: ptr [[PTR:%.*]]) {
; CHECK-NEXT: [[LD0:%.*]] = load i64, ptr [[PTR]], align 16
; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 16
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr [[GEP2]], align 16
-; CHECK-NEXT: [[LD21:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; CHECK-NEXT: [[LD32:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b64>, ptr [[GEP2]], align 16
+; CHECK-NEXT: [[LD21:%.*]] = extractelement <2 x b64> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[LD21]] to i64
+; CHECK-NEXT: [[LD32:%.*]] = extractelement <2 x b64> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b64 [[LD32]] to i64
; CHECK-NEXT: ret void
;
%ld0 = load i64, ptr %ptr, align 16
@@ -83,11 +95,15 @@ define void @test4(ptr %ptr) {
define void @test5(ptr %ptr) {
; CHECK-LABEL: define void @test5(
; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> poison)
-; CHECK-NEXT: [[LD03:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[GAPFILL4:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[GAPFILL25:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[LD36:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x b32> poison)
+; CHECK-NEXT: [[LD03:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD03]] to i32
+; CHECK-NEXT: [[GAPFILL4:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[GAPFILL4]] to i32
+; CHECK-NEXT: [[GAPFILL25:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[GAPFILL25]] to i32
+; CHECK-NEXT: [[LD36:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LD36]] to i32
; CHECK-NEXT: ret void
;
%ld0 = load i32, ptr %ptr, align 16
@@ -100,17 +116,25 @@ define void @test5(ptr %ptr) {
define void @test6(ptr %ptr) {
; CHECK-LABEL: define void @test6(
; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
-; CHECK-NEXT: [[LD03:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LD14:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[GAPFILL5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[LD36:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x b32> poison)
+; CHECK-NEXT: [[LD03:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD03]] to i32
+; CHECK-NEXT: [[LD14:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD14]] to i32
+; CHECK-NEXT: [[GAPFILL5:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[GAPFILL5]] to i32
+; CHECK-NEXT: [[LD36:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LD36]] to i32
; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 16
-; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[GEP4]], <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x i32> poison)
-; CHECK-NEXT: [[LD47:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
-; CHECK-NEXT: [[GAPFILL28:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
-; CHECK-NEXT: [[LD69:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
-; CHECK-NEXT: [[LD710:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[GEP4]], <4 x i1> <i1 true, i1 false, i1 true, i1 true>, <4 x b32> poison)
+; CHECK-NEXT: [[LD47:%.*]] = extractelement <4 x b32> [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b32 [[LD47]] to i32
+; CHECK-NEXT: [[GAPFILL28:%.*]] = extractelement <4 x b32> [[TMP6]], i32 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b32 [[GAPFILL28]] to i32
+; CHECK-NEXT: [[LD69:%.*]] = extractelement <4 x b32> [[TMP6]], i32 2
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast b32 [[LD69]] to i32
+; CHECK-NEXT: [[LD710:%.*]] = extractelement <4 x b32> [[TMP6]], i32 3
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast b32 [[LD710]] to i32
; CHECK-NEXT: ret void
;
%ld0 = load i32, ptr %ptr, align 16
@@ -132,17 +156,25 @@ define void @test6(ptr %ptr) {
define void @test7(ptr %ptr) {
; CHECK-LABEL: define void @test7(
; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
-; CHECK-NEXT: [[LD05:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LD16:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[GAPFILL7:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[LD38:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x b32> poison)
+; CHECK-NEXT: [[LD05:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD05]] to i32
+; CHECK-NEXT: [[LD16:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD16]] to i32
+; CHECK-NEXT: [[GAPFILL7:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[GAPFILL7]] to i32
+; CHECK-NEXT: [[LD38:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LD38]] to i32
; CHECK-NEXT: [[GEP4:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 16
-; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[GEP4]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x i32> poison)
-; CHECK-NEXT: [[LD49:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
-; CHECK-NEXT: [[GAPFILL210:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
-; CHECK-NEXT: [[GAPFILL411:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
-; CHECK-NEXT: [[LD712:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[GEP4]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>, <4 x b32> poison)
+; CHECK-NEXT: [[LD49:%.*]] = extractelement <4 x b32> [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b32 [[LD49]] to i32
+; CHECK-NEXT: [[GAPFILL210:%.*]] = extractelement <4 x b32> [[TMP6]], i32 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b32 [[GAPFILL210]] to i32
+; CHECK-NEXT: [[GAPFILL411:%.*]] = extractelement <4 x b32> [[TMP6]], i32 2
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast b32 [[GAPFILL411]] to i32
+; CHECK-NEXT: [[LD712:%.*]] = extractelement <4 x b32> [[TMP6]], i32 3
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast b32 [[LD712]] to i32
; CHECK-NEXT: ret void
;
%ld0 = load i32, ptr %ptr, align 16
@@ -164,18 +196,26 @@ define void @test7(ptr %ptr) {
define void @test8(ptr %ptr) {
; CHECK-LABEL: define void @test8(
; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x i32> poison)
-; CHECK-NEXT: [[LD03:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LD14:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[GAPFILL5:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[LD36:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[PTR]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>, <4 x b32> poison)
+; CHECK-NEXT: [[LD03:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD03]] to i32
+; CHECK-NEXT: [[LD14:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD14]] to i32
+; CHECK-NEXT: [[GAPFILL5:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[GAPFILL5]] to i32
+; CHECK-NEXT: [[LD36:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LD36]] to i32
; CHECK-NEXT: [[GEP3:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 12
; CHECK-NEXT: [[GAPFILLGEP1:%.*]] = getelementptr i8, ptr [[GEP3]], i64 4
-; CHECK-NEXT: [[TMP2:%.*]] = call <4 x i32> @llvm.masked.load.v4i32.p0(ptr align 16 [[GAPFILLGEP1]], <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x i32> poison)
-; CHECK-NEXT: [[GAPFILL27:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
-; CHECK-NEXT: [[LD58:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
-; CHECK-NEXT: [[LD69:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
-; CHECK-NEXT: [[LD710:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
+; CHECK-NEXT: [[TMP6:%.*]] = call <4 x b32> @llvm.masked.load.v4b32.p0(ptr align 16 [[GAPFILLGEP1]], <4 x i1> <i1 false, i1 true, i1 true, i1 true>, <4 x b32> poison)
+; CHECK-NEXT: [[GAPFILL27:%.*]] = extractelement <4 x b32> [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b32 [[GAPFILL27]] to i32
+; CHECK-NEXT: [[LD58:%.*]] = extractelement <4 x b32> [[TMP6]], i32 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b32 [[LD58]] to i32
+; CHECK-NEXT: [[LD69:%.*]] = extractelement <4 x b32> [[TMP6]], i32 2
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast b32 [[LD69]] to i32
+; CHECK-NEXT: [[LD710:%.*]] = extractelement <4 x b32> [[TMP6]], i32 3
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast b32 [[LD710]] to i32
; CHECK-NEXT: ret void
;
%ld0 = load i32, ptr %ptr, align 16
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/load-store-256-bit.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/load-store-256-bit.ll
index 649347d455c67..173d6b6788b74 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/load-store-256-bit.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/load-store-256-bit.ll
@@ -21,145 +21,273 @@ define void @int8x32(ptr addrspace(1) %ptr) {
; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
; SM90-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr addrspace(1) [[PTR]], i64 0
; SM90-NEXT: [[PTR10:%.*]] = getelementptr i8, ptr addrspace(1) [[PTR]], i64 16
-; SM90-NEXT: [[TMP1:%.*]] = load <16 x i8>, ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[L01:%.*]] = extractelement <16 x i8> [[TMP1]], i32 0
-; SM90-NEXT: [[L110:%.*]] = extractelement <16 x i8> [[TMP1]], i32 1
-; SM90-NEXT: [[L211:%.*]] = extractelement <16 x i8> [[TMP1]], i32 2
-; SM90-NEXT: [[L312:%.*]] = extractelement <16 x i8> [[TMP1]], i32 3
-; SM90-NEXT: [[L413:%.*]] = extractelement <16 x i8> [[TMP1]], i32 4
-; SM90-NEXT: [[L514:%.*]] = extractelement <16 x i8> [[TMP1]], i32 5
-; SM90-NEXT: [[L615:%.*]] = extractelement <16 x i8> [[TMP1]], i32 6
-; SM90-NEXT: [[L716:%.*]] = extractelement <16 x i8> [[TMP1]], i32 7
-; SM90-NEXT: [[L817:%.*]] = extractelement <16 x i8> [[TMP1]], i32 8
-; SM90-NEXT: [[L918:%.*]] = extractelement <16 x i8> [[TMP1]], i32 9
-; SM90-NEXT: [[LA19:%.*]] = extractelement <16 x i8> [[TMP1]], i32 10
-; SM90-NEXT: [[LB20:%.*]] = extractelement <16 x i8> [[TMP1]], i32 11
-; SM90-NEXT: [[LC21:%.*]] = extractelement <16 x i8> [[TMP1]], i32 12
-; SM90-NEXT: [[LD22:%.*]] = extractelement <16 x i8> [[TMP1]], i32 13
-; SM90-NEXT: [[LE23:%.*]] = extractelement <16 x i8> [[TMP1]], i32 14
-; SM90-NEXT: [[LF24:%.*]] = extractelement <16 x i8> [[TMP1]], i32 15
-; SM90-NEXT: [[TMP2:%.*]] = load <16 x i8>, ptr addrspace(1) [[PTR10]], align 16
-; SM90-NEXT: [[L1025:%.*]] = extractelement <16 x i8> [[TMP2]], i32 0
-; SM90-NEXT: [[L1126:%.*]] = extractelement <16 x i8> [[TMP2]], i32 1
-; SM90-NEXT: [[L1227:%.*]] = extractelement <16 x i8> [[TMP2]], i32 2
-; SM90-NEXT: [[L1328:%.*]] = extractelement <16 x i8> [[TMP2]], i32 3
-; SM90-NEXT: [[L1429:%.*]] = extractelement <16 x i8> [[TMP2]], i32 4
-; SM90-NEXT: [[L1530:%.*]] = extractelement <16 x i8> [[TMP2]], i32 5
-; SM90-NEXT: [[L1631:%.*]] = extractelement <16 x i8> [[TMP2]], i32 6
-; SM90-NEXT: [[L1732:%.*]] = extractelement <16 x i8> [[TMP2]], i32 7
-; SM90-NEXT: [[L1833:%.*]] = extractelement <16 x i8> [[TMP2]], i32 8
-; SM90-NEXT: [[L1934:%.*]] = extractelement <16 x i8> [[TMP2]], i32 9
-; SM90-NEXT: [[L1A35:%.*]] = extractelement <16 x i8> [[TMP2]], i32 10
-; SM90-NEXT: [[L1B36:%.*]] = extractelement <16 x i8> [[TMP2]], i32 11
-; SM90-NEXT: [[L1C37:%.*]] = extractelement <16 x i8> [[TMP2]], i32 12
-; SM90-NEXT: [[L1D38:%.*]] = extractelement <16 x i8> [[TMP2]], i32 13
-; SM90-NEXT: [[L1E39:%.*]] = extractelement <16 x i8> [[TMP2]], i32 14
-; SM90-NEXT: [[L1F40:%.*]] = extractelement <16 x i8> [[TMP2]], i32 15
-; SM90-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> poison, i8 [[LB20]], i32 0
-; SM90-NEXT: [[TMP4:%.*]] = insertelement <16 x i8> [[TMP3]], i8 [[LA19]], i32 1
-; SM90-NEXT: [[TMP5:%.*]] = insertelement <16 x i8> [[TMP4]], i8 [[L918]], i32 2
-; SM90-NEXT: [[TMP6:%.*]] = insertelement <16 x i8> [[TMP5]], i8 [[L817]], i32 3
-; SM90-NEXT: [[TMP7:%.*]] = insertelement <16 x i8> [[TMP6]], i8 [[L716]], i32 4
-; SM90-NEXT: [[TMP8:%.*]] = insertelement <16 x i8> [[TMP7]], i8 [[L615]], i32 5
-; SM90-NEXT: [[TMP9:%.*]] = insertelement <16 x i8> [[TMP8]], i8 [[L514]], i32 6
-; SM90-NEXT: [[TMP10:%.*]] = insertelement <16 x i8> [[TMP9]], i8 [[L413]], i32 7
-; SM90-NEXT: [[TMP11:%.*]] = insertelement <16 x i8> [[TMP10]], i8 [[L312]], i32 8
-; SM90-NEXT: [[TMP12:%.*]] = insertelement <16 x i8> [[TMP11]], i8 [[L211]], i32 9
-; SM90-NEXT: [[TMP13:%.*]] = insertelement <16 x i8> [[TMP12]], i8 [[L110]], i32 10
-; SM90-NEXT: [[TMP14:%.*]] = insertelement <16 x i8> [[TMP13]], i8 [[L01]], i32 11
-; SM90-NEXT: [[TMP15:%.*]] = insertelement <16 x i8> [[TMP14]], i8 [[LF24]], i32 12
-; SM90-NEXT: [[TMP16:%.*]] = insertelement <16 x i8> [[TMP15]], i8 [[LE23]], i32 13
-; SM90-NEXT: [[TMP17:%.*]] = insertelement <16 x i8> [[TMP16]], i8 [[LD22]], i32 14
-; SM90-NEXT: [[TMP18:%.*]] = insertelement <16 x i8> [[TMP17]], i8 [[LC21]], i32 15
-; SM90-NEXT: store <16 x i8> [[TMP18]], ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[TMP19:%.*]] = insertelement <16 x i8> poison, i8 [[L1B36]], i32 0
-; SM90-NEXT: [[TMP20:%.*]] = insertelement <16 x i8> [[TMP19]], i8 [[L1A35]], i32 1
-; SM90-NEXT: [[TMP21:%.*]] = insertelement <16 x i8> [[TMP20]], i8 [[L1934]], i32 2
-; SM90-NEXT: [[TMP22:%.*]] = insertelement <16 x i8> [[TMP21]], i8 [[L1833]], i32 3
-; SM90-NEXT: [[TMP23:%.*]] = insertelement <16 x i8> [[TMP22]], i8 [[L1732]], i32 4
-; SM90-NEXT: [[TMP24:%.*]] = insertelement <16 x i8> [[TMP23]], i8 [[L1631]], i32 5
-; SM90-NEXT: [[TMP25:%.*]] = insertelement <16 x i8> [[TMP24]], i8 [[L1530]], i32 6
-; SM90-NEXT: [[TMP26:%.*]] = insertelement <16 x i8> [[TMP25]], i8 [[L1429]], i32 7
-; SM90-NEXT: [[TMP27:%.*]] = insertelement <16 x i8> [[TMP26]], i8 [[L1328]], i32 8
-; SM90-NEXT: [[TMP28:%.*]] = insertelement <16 x i8> [[TMP27]], i8 [[L1227]], i32 9
-; SM90-NEXT: [[TMP29:%.*]] = insertelement <16 x i8> [[TMP28]], i8 [[L1126]], i32 10
-; SM90-NEXT: [[TMP30:%.*]] = insertelement <16 x i8> [[TMP29]], i8 [[L1025]], i32 11
-; SM90-NEXT: [[TMP31:%.*]] = insertelement <16 x i8> [[TMP30]], i8 [[L1F40]], i32 12
-; SM90-NEXT: [[TMP32:%.*]] = insertelement <16 x i8> [[TMP31]], i8 [[L1E39]], i32 13
-; SM90-NEXT: [[TMP33:%.*]] = insertelement <16 x i8> [[TMP32]], i8 [[L1D38]], i32 14
-; SM90-NEXT: [[TMP34:%.*]] = insertelement <16 x i8> [[TMP33]], i8 [[L1C37]], i32 15
-; SM90-NEXT: store <16 x i8> [[TMP34]], ptr addrspace(1) [[PTR10]], align 16
+; SM90-NEXT: [[TMP1:%.*]] = load <16 x b8>, ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[L01:%.*]] = extractelement <16 x b8> [[TMP1]], i32 0
+; SM90-NEXT: [[TMP2:%.*]] = bitcast b8 [[L01]] to i8
+; SM90-NEXT: [[L110:%.*]] = extractelement <16 x b8> [[TMP1]], i32 1
+; SM90-NEXT: [[TMP3:%.*]] = bitcast b8 [[L110]] to i8
+; SM90-NEXT: [[L211:%.*]] = extractelement <16 x b8> [[TMP1]], i32 2
+; SM90-NEXT: [[TMP4:%.*]] = bitcast b8 [[L211]] to i8
+; SM90-NEXT: [[L312:%.*]] = extractelement <16 x b8> [[TMP1]], i32 3
+; SM90-NEXT: [[TMP5:%.*]] = bitcast b8 [[L312]] to i8
+; SM90-NEXT: [[L413:%.*]] = extractelement <16 x b8> [[TMP1]], i32 4
+; SM90-NEXT: [[TMP6:%.*]] = bitcast b8 [[L413]] to i8
+; SM90-NEXT: [[L514:%.*]] = extractelement <16 x b8> [[TMP1]], i32 5
+; SM90-NEXT: [[TMP7:%.*]] = bitcast b8 [[L514]] to i8
+; SM90-NEXT: [[L615:%.*]] = extractelement <16 x b8> [[TMP1]], i32 6
+; SM90-NEXT: [[TMP8:%.*]] = bitcast b8 [[L615]] to i8
+; SM90-NEXT: [[L716:%.*]] = extractelement <16 x b8> [[TMP1]], i32 7
+; SM90-NEXT: [[TMP9:%.*]] = bitcast b8 [[L716]] to i8
+; SM90-NEXT: [[L817:%.*]] = extractelement <16 x b8> [[TMP1]], i32 8
+; SM90-NEXT: [[TMP10:%.*]] = bitcast b8 [[L817]] to i8
+; SM90-NEXT: [[L918:%.*]] = extractelement <16 x b8> [[TMP1]], i32 9
+; SM90-NEXT: [[TMP11:%.*]] = bitcast b8 [[L918]] to i8
+; SM90-NEXT: [[LA19:%.*]] = extractelement <16 x b8> [[TMP1]], i32 10
+; SM90-NEXT: [[TMP12:%.*]] = bitcast b8 [[LA19]] to i8
+; SM90-NEXT: [[LB20:%.*]] = extractelement <16 x b8> [[TMP1]], i32 11
+; SM90-NEXT: [[TMP13:%.*]] = bitcast b8 [[LB20]] to i8
+; SM90-NEXT: [[LC21:%.*]] = extractelement <16 x b8> [[TMP1]], i32 12
+; SM90-NEXT: [[TMP14:%.*]] = bitcast b8 [[LC21]] to i8
+; SM90-NEXT: [[LD22:%.*]] = extractelement <16 x b8> [[TMP1]], i32 13
+; SM90-NEXT: [[TMP15:%.*]] = bitcast b8 [[LD22]] to i8
+; SM90-NEXT: [[LE23:%.*]] = extractelement <16 x b8> [[TMP1]], i32 14
+; SM90-NEXT: [[TMP16:%.*]] = bitcast b8 [[LE23]] to i8
+; SM90-NEXT: [[LF24:%.*]] = extractelement <16 x b8> [[TMP1]], i32 15
+; SM90-NEXT: [[TMP17:%.*]] = bitcast b8 [[LF24]] to i8
+; SM90-NEXT: [[TMP18:%.*]] = load <16 x b8>, ptr addrspace(1) [[PTR10]], align 16
+; SM90-NEXT: [[L1025:%.*]] = extractelement <16 x b8> [[TMP18]], i32 0
+; SM90-NEXT: [[TMP19:%.*]] = bitcast b8 [[L1025]] to i8
+; SM90-NEXT: [[L1126:%.*]] = extractelement <16 x b8> [[TMP18]], i32 1
+; SM90-NEXT: [[TMP20:%.*]] = bitcast b8 [[L1126]] to i8
+; SM90-NEXT: [[L1227:%.*]] = extractelement <16 x b8> [[TMP18]], i32 2
+; SM90-NEXT: [[TMP21:%.*]] = bitcast b8 [[L1227]] to i8
+; SM90-NEXT: [[L1328:%.*]] = extractelement <16 x b8> [[TMP18]], i32 3
+; SM90-NEXT: [[TMP22:%.*]] = bitcast b8 [[L1328]] to i8
+; SM90-NEXT: [[L1429:%.*]] = extractelement <16 x b8> [[TMP18]], i32 4
+; SM90-NEXT: [[TMP23:%.*]] = bitcast b8 [[L1429]] to i8
+; SM90-NEXT: [[L1530:%.*]] = extractelement <16 x b8> [[TMP18]], i32 5
+; SM90-NEXT: [[TMP24:%.*]] = bitcast b8 [[L1530]] to i8
+; SM90-NEXT: [[L1631:%.*]] = extractelement <16 x b8> [[TMP18]], i32 6
+; SM90-NEXT: [[TMP25:%.*]] = bitcast b8 [[L1631]] to i8
+; SM90-NEXT: [[L1732:%.*]] = extractelement <16 x b8> [[TMP18]], i32 7
+; SM90-NEXT: [[TMP26:%.*]] = bitcast b8 [[L1732]] to i8
+; SM90-NEXT: [[L1833:%.*]] = extractelement <16 x b8> [[TMP18]], i32 8
+; SM90-NEXT: [[TMP27:%.*]] = bitcast b8 [[L1833]] to i8
+; SM90-NEXT: [[L1934:%.*]] = extractelement <16 x b8> [[TMP18]], i32 9
+; SM90-NEXT: [[TMP28:%.*]] = bitcast b8 [[L1934]] to i8
+; SM90-NEXT: [[L1A35:%.*]] = extractelement <16 x b8> [[TMP18]], i32 10
+; SM90-NEXT: [[TMP29:%.*]] = bitcast b8 [[L1A35]] to i8
+; SM90-NEXT: [[L1B36:%.*]] = extractelement <16 x b8> [[TMP18]], i32 11
+; SM90-NEXT: [[TMP30:%.*]] = bitcast b8 [[L1B36]] to i8
+; SM90-NEXT: [[L1C37:%.*]] = extractelement <16 x b8> [[TMP18]], i32 12
+; SM90-NEXT: [[TMP31:%.*]] = bitcast b8 [[L1C37]] to i8
+; SM90-NEXT: [[L1D38:%.*]] = extractelement <16 x b8> [[TMP18]], i32 13
+; SM90-NEXT: [[TMP32:%.*]] = bitcast b8 [[L1D38]] to i8
+; SM90-NEXT: [[L1E39:%.*]] = extractelement <16 x b8> [[TMP18]], i32 14
+; SM90-NEXT: [[TMP33:%.*]] = bitcast b8 [[L1E39]] to i8
+; SM90-NEXT: [[L1F40:%.*]] = extractelement <16 x b8> [[TMP18]], i32 15
+; SM90-NEXT: [[TMP34:%.*]] = bitcast b8 [[L1F40]] to i8
+; SM90-NEXT: [[TMP35:%.*]] = bitcast i8 [[TMP13]] to b8
+; SM90-NEXT: [[TMP36:%.*]] = insertelement <16 x b8> poison, b8 [[TMP35]], i32 0
+; SM90-NEXT: [[TMP37:%.*]] = bitcast i8 [[TMP12]] to b8
+; SM90-NEXT: [[TMP38:%.*]] = insertelement <16 x b8> [[TMP36]], b8 [[TMP37]], i32 1
+; SM90-NEXT: [[TMP39:%.*]] = bitcast i8 [[TMP11]] to b8
+; SM90-NEXT: [[TMP40:%.*]] = insertelement <16 x b8> [[TMP38]], b8 [[TMP39]], i32 2
+; SM90-NEXT: [[TMP41:%.*]] = bitcast i8 [[TMP10]] to b8
+; SM90-NEXT: [[TMP42:%.*]] = insertelement <16 x b8> [[TMP40]], b8 [[TMP41]], i32 3
+; SM90-NEXT: [[TMP43:%.*]] = bitcast i8 [[TMP9]] to b8
+; SM90-NEXT: [[TMP44:%.*]] = insertelement <16 x b8> [[TMP42]], b8 [[TMP43]], i32 4
+; SM90-NEXT: [[TMP45:%.*]] = bitcast i8 [[TMP8]] to b8
+; SM90-NEXT: [[TMP46:%.*]] = insertelement <16 x b8> [[TMP44]], b8 [[TMP45]], i32 5
+; SM90-NEXT: [[TMP47:%.*]] = bitcast i8 [[TMP7]] to b8
+; SM90-NEXT: [[TMP48:%.*]] = insertelement <16 x b8> [[TMP46]], b8 [[TMP47]], i32 6
+; SM90-NEXT: [[TMP49:%.*]] = bitcast i8 [[TMP6]] to b8
+; SM90-NEXT: [[TMP50:%.*]] = insertelement <16 x b8> [[TMP48]], b8 [[TMP49]], i32 7
+; SM90-NEXT: [[TMP51:%.*]] = bitcast i8 [[TMP5]] to b8
+; SM90-NEXT: [[TMP52:%.*]] = insertelement <16 x b8> [[TMP50]], b8 [[TMP51]], i32 8
+; SM90-NEXT: [[TMP53:%.*]] = bitcast i8 [[TMP4]] to b8
+; SM90-NEXT: [[TMP54:%.*]] = insertelement <16 x b8> [[TMP52]], b8 [[TMP53]], i32 9
+; SM90-NEXT: [[TMP55:%.*]] = bitcast i8 [[TMP3]] to b8
+; SM90-NEXT: [[TMP56:%.*]] = insertelement <16 x b8> [[TMP54]], b8 [[TMP55]], i32 10
+; SM90-NEXT: [[TMP57:%.*]] = bitcast i8 [[TMP2]] to b8
+; SM90-NEXT: [[TMP58:%.*]] = insertelement <16 x b8> [[TMP56]], b8 [[TMP57]], i32 11
+; SM90-NEXT: [[TMP59:%.*]] = bitcast i8 [[TMP17]] to b8
+; SM90-NEXT: [[TMP60:%.*]] = insertelement <16 x b8> [[TMP58]], b8 [[TMP59]], i32 12
+; SM90-NEXT: [[TMP61:%.*]] = bitcast i8 [[TMP16]] to b8
+; SM90-NEXT: [[TMP62:%.*]] = insertelement <16 x b8> [[TMP60]], b8 [[TMP61]], i32 13
+; SM90-NEXT: [[TMP63:%.*]] = bitcast i8 [[TMP15]] to b8
+; SM90-NEXT: [[TMP64:%.*]] = insertelement <16 x b8> [[TMP62]], b8 [[TMP63]], i32 14
+; SM90-NEXT: [[TMP65:%.*]] = bitcast i8 [[TMP14]] to b8
+; SM90-NEXT: [[TMP66:%.*]] = insertelement <16 x b8> [[TMP64]], b8 [[TMP65]], i32 15
+; SM90-NEXT: store <16 x b8> [[TMP66]], ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[TMP67:%.*]] = bitcast i8 [[TMP30]] to b8
+; SM90-NEXT: [[TMP68:%.*]] = insertelement <16 x b8> poison, b8 [[TMP67]], i32 0
+; SM90-NEXT: [[TMP69:%.*]] = bitcast i8 [[TMP29]] to b8
+; SM90-NEXT: [[TMP70:%.*]] = insertelement <16 x b8> [[TMP68]], b8 [[TMP69]], i32 1
+; SM90-NEXT: [[TMP71:%.*]] = bitcast i8 [[TMP28]] to b8
+; SM90-NEXT: [[TMP72:%.*]] = insertelement <16 x b8> [[TMP70]], b8 [[TMP71]], i32 2
+; SM90-NEXT: [[TMP73:%.*]] = bitcast i8 [[TMP27]] to b8
+; SM90-NEXT: [[TMP74:%.*]] = insertelement <16 x b8> [[TMP72]], b8 [[TMP73]], i32 3
+; SM90-NEXT: [[TMP75:%.*]] = bitcast i8 [[TMP26]] to b8
+; SM90-NEXT: [[TMP76:%.*]] = insertelement <16 x b8> [[TMP74]], b8 [[TMP75]], i32 4
+; SM90-NEXT: [[TMP77:%.*]] = bitcast i8 [[TMP25]] to b8
+; SM90-NEXT: [[TMP78:%.*]] = insertelement <16 x b8> [[TMP76]], b8 [[TMP77]], i32 5
+; SM90-NEXT: [[TMP79:%.*]] = bitcast i8 [[TMP24]] to b8
+; SM90-NEXT: [[TMP80:%.*]] = insertelement <16 x b8> [[TMP78]], b8 [[TMP79]], i32 6
+; SM90-NEXT: [[TMP81:%.*]] = bitcast i8 [[TMP23]] to b8
+; SM90-NEXT: [[TMP82:%.*]] = insertelement <16 x b8> [[TMP80]], b8 [[TMP81]], i32 7
+; SM90-NEXT: [[TMP83:%.*]] = bitcast i8 [[TMP22]] to b8
+; SM90-NEXT: [[TMP84:%.*]] = insertelement <16 x b8> [[TMP82]], b8 [[TMP83]], i32 8
+; SM90-NEXT: [[TMP85:%.*]] = bitcast i8 [[TMP21]] to b8
+; SM90-NEXT: [[TMP86:%.*]] = insertelement <16 x b8> [[TMP84]], b8 [[TMP85]], i32 9
+; SM90-NEXT: [[TMP87:%.*]] = bitcast i8 [[TMP20]] to b8
+; SM90-NEXT: [[TMP88:%.*]] = insertelement <16 x b8> [[TMP86]], b8 [[TMP87]], i32 10
+; SM90-NEXT: [[TMP89:%.*]] = bitcast i8 [[TMP19]] to b8
+; SM90-NEXT: [[TMP90:%.*]] = insertelement <16 x b8> [[TMP88]], b8 [[TMP89]], i32 11
+; SM90-NEXT: [[TMP91:%.*]] = bitcast i8 [[TMP34]] to b8
+; SM90-NEXT: [[TMP92:%.*]] = insertelement <16 x b8> [[TMP90]], b8 [[TMP91]], i32 12
+; SM90-NEXT: [[TMP93:%.*]] = bitcast i8 [[TMP33]] to b8
+; SM90-NEXT: [[TMP94:%.*]] = insertelement <16 x b8> [[TMP92]], b8 [[TMP93]], i32 13
+; SM90-NEXT: [[TMP95:%.*]] = bitcast i8 [[TMP32]] to b8
+; SM90-NEXT: [[TMP96:%.*]] = insertelement <16 x b8> [[TMP94]], b8 [[TMP95]], i32 14
+; SM90-NEXT: [[TMP97:%.*]] = bitcast i8 [[TMP31]] to b8
+; SM90-NEXT: [[TMP98:%.*]] = insertelement <16 x b8> [[TMP96]], b8 [[TMP97]], i32 15
+; SM90-NEXT: store <16 x b8> [[TMP98]], ptr addrspace(1) [[PTR10]], align 16
; SM90-NEXT: ret void
;
; SM100-LABEL: define void @int8x32(
; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
; SM100-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr addrspace(1) [[PTR]], i64 0
-; SM100-NEXT: [[TMP1:%.*]] = load <32 x i8>, ptr addrspace(1) [[PTR0]], align 32
-; SM100-NEXT: [[L01:%.*]] = extractelement <32 x i8> [[TMP1]], i32 0
-; SM100-NEXT: [[L110:%.*]] = extractelement <32 x i8> [[TMP1]], i32 1
-; SM100-NEXT: [[L211:%.*]] = extractelement <32 x i8> [[TMP1]], i32 2
-; SM100-NEXT: [[L312:%.*]] = extractelement <32 x i8> [[TMP1]], i32 3
-; SM100-NEXT: [[L413:%.*]] = extractelement <32 x i8> [[TMP1]], i32 4
-; SM100-NEXT: [[L514:%.*]] = extractelement <32 x i8> [[TMP1]], i32 5
-; SM100-NEXT: [[L615:%.*]] = extractelement <32 x i8> [[TMP1]], i32 6
-; SM100-NEXT: [[L716:%.*]] = extractelement <32 x i8> [[TMP1]], i32 7
-; SM100-NEXT: [[L817:%.*]] = extractelement <32 x i8> [[TMP1]], i32 8
-; SM100-NEXT: [[L918:%.*]] = extractelement <32 x i8> [[TMP1]], i32 9
-; SM100-NEXT: [[LA19:%.*]] = extractelement <32 x i8> [[TMP1]], i32 10
-; SM100-NEXT: [[LB20:%.*]] = extractelement <32 x i8> [[TMP1]], i32 11
-; SM100-NEXT: [[LC21:%.*]] = extractelement <32 x i8> [[TMP1]], i32 12
-; SM100-NEXT: [[LD22:%.*]] = extractelement <32 x i8> [[TMP1]], i32 13
-; SM100-NEXT: [[LE23:%.*]] = extractelement <32 x i8> [[TMP1]], i32 14
-; SM100-NEXT: [[LF24:%.*]] = extractelement <32 x i8> [[TMP1]], i32 15
-; SM100-NEXT: [[L1025:%.*]] = extractelement <32 x i8> [[TMP1]], i32 16
-; SM100-NEXT: [[L1126:%.*]] = extractelement <32 x i8> [[TMP1]], i32 17
-; SM100-NEXT: [[L1227:%.*]] = extractelement <32 x i8> [[TMP1]], i32 18
-; SM100-NEXT: [[L1328:%.*]] = extractelement <32 x i8> [[TMP1]], i32 19
-; SM100-NEXT: [[L1429:%.*]] = extractelement <32 x i8> [[TMP1]], i32 20
-; SM100-NEXT: [[L1530:%.*]] = extractelement <32 x i8> [[TMP1]], i32 21
-; SM100-NEXT: [[L1631:%.*]] = extractelement <32 x i8> [[TMP1]], i32 22
-; SM100-NEXT: [[L1732:%.*]] = extractelement <32 x i8> [[TMP1]], i32 23
-; SM100-NEXT: [[L1833:%.*]] = extractelement <32 x i8> [[TMP1]], i32 24
-; SM100-NEXT: [[L1934:%.*]] = extractelement <32 x i8> [[TMP1]], i32 25
-; SM100-NEXT: [[L1A35:%.*]] = extractelement <32 x i8> [[TMP1]], i32 26
-; SM100-NEXT: [[L1B36:%.*]] = extractelement <32 x i8> [[TMP1]], i32 27
-; SM100-NEXT: [[L1C37:%.*]] = extractelement <32 x i8> [[TMP1]], i32 28
-; SM100-NEXT: [[L1D38:%.*]] = extractelement <32 x i8> [[TMP1]], i32 29
-; SM100-NEXT: [[L1E39:%.*]] = extractelement <32 x i8> [[TMP1]], i32 30
-; SM100-NEXT: [[L1F40:%.*]] = extractelement <32 x i8> [[TMP1]], i32 31
-; SM100-NEXT: [[TMP2:%.*]] = insertelement <32 x i8> poison, i8 [[LB20]], i32 0
-; SM100-NEXT: [[TMP3:%.*]] = insertelement <32 x i8> [[TMP2]], i8 [[LA19]], i32 1
-; SM100-NEXT: [[TMP4:%.*]] = insertelement <32 x i8> [[TMP3]], i8 [[L918]], i32 2
-; SM100-NEXT: [[TMP5:%.*]] = insertelement <32 x i8> [[TMP4]], i8 [[L817]], i32 3
-; SM100-NEXT: [[TMP6:%.*]] = insertelement <32 x i8> [[TMP5]], i8 [[L716]], i32 4
-; SM100-NEXT: [[TMP7:%.*]] = insertelement <32 x i8> [[TMP6]], i8 [[L615]], i32 5
-; SM100-NEXT: [[TMP8:%.*]] = insertelement <32 x i8> [[TMP7]], i8 [[L514]], i32 6
-; SM100-NEXT: [[TMP9:%.*]] = insertelement <32 x i8> [[TMP8]], i8 [[L413]], i32 7
-; SM100-NEXT: [[TMP10:%.*]] = insertelement <32 x i8> [[TMP9]], i8 [[L312]], i32 8
-; SM100-NEXT: [[TMP11:%.*]] = insertelement <32 x i8> [[TMP10]], i8 [[L211]], i32 9
-; SM100-NEXT: [[TMP12:%.*]] = insertelement <32 x i8> [[TMP11]], i8 [[L110]], i32 10
-; SM100-NEXT: [[TMP13:%.*]] = insertelement <32 x i8> [[TMP12]], i8 [[L01]], i32 11
-; SM100-NEXT: [[TMP14:%.*]] = insertelement <32 x i8> [[TMP13]], i8 [[LF24]], i32 12
-; SM100-NEXT: [[TMP15:%.*]] = insertelement <32 x i8> [[TMP14]], i8 [[LE23]], i32 13
-; SM100-NEXT: [[TMP16:%.*]] = insertelement <32 x i8> [[TMP15]], i8 [[LD22]], i32 14
-; SM100-NEXT: [[TMP17:%.*]] = insertelement <32 x i8> [[TMP16]], i8 [[LC21]], i32 15
-; SM100-NEXT: [[TMP18:%.*]] = insertelement <32 x i8> [[TMP17]], i8 [[L1B36]], i32 16
-; SM100-NEXT: [[TMP19:%.*]] = insertelement <32 x i8> [[TMP18]], i8 [[L1A35]], i32 17
-; SM100-NEXT: [[TMP20:%.*]] = insertelement <32 x i8> [[TMP19]], i8 [[L1934]], i32 18
-; SM100-NEXT: [[TMP21:%.*]] = insertelement <32 x i8> [[TMP20]], i8 [[L1833]], i32 19
-; SM100-NEXT: [[TMP22:%.*]] = insertelement <32 x i8> [[TMP21]], i8 [[L1732]], i32 20
-; SM100-NEXT: [[TMP23:%.*]] = insertelement <32 x i8> [[TMP22]], i8 [[L1631]], i32 21
-; SM100-NEXT: [[TMP24:%.*]] = insertelement <32 x i8> [[TMP23]], i8 [[L1530]], i32 22
-; SM100-NEXT: [[TMP25:%.*]] = insertelement <32 x i8> [[TMP24]], i8 [[L1429]], i32 23
-; SM100-NEXT: [[TMP26:%.*]] = insertelement <32 x i8> [[TMP25]], i8 [[L1328]], i32 24
-; SM100-NEXT: [[TMP27:%.*]] = insertelement <32 x i8> [[TMP26]], i8 [[L1227]], i32 25
-; SM100-NEXT: [[TMP28:%.*]] = insertelement <32 x i8> [[TMP27]], i8 [[L1126]], i32 26
-; SM100-NEXT: [[TMP29:%.*]] = insertelement <32 x i8> [[TMP28]], i8 [[L1025]], i32 27
-; SM100-NEXT: [[TMP30:%.*]] = insertelement <32 x i8> [[TMP29]], i8 [[L1F40]], i32 28
-; SM100-NEXT: [[TMP31:%.*]] = insertelement <32 x i8> [[TMP30]], i8 [[L1E39]], i32 29
-; SM100-NEXT: [[TMP32:%.*]] = insertelement <32 x i8> [[TMP31]], i8 [[L1D38]], i32 30
-; SM100-NEXT: [[TMP33:%.*]] = insertelement <32 x i8> [[TMP32]], i8 [[L1C37]], i32 31
-; SM100-NEXT: store <32 x i8> [[TMP33]], ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[TMP1:%.*]] = load <32 x b8>, ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[L01:%.*]] = extractelement <32 x b8> [[TMP1]], i32 0
+; SM100-NEXT: [[TMP2:%.*]] = bitcast b8 [[L01]] to i8
+; SM100-NEXT: [[L110:%.*]] = extractelement <32 x b8> [[TMP1]], i32 1
+; SM100-NEXT: [[TMP3:%.*]] = bitcast b8 [[L110]] to i8
+; SM100-NEXT: [[L211:%.*]] = extractelement <32 x b8> [[TMP1]], i32 2
+; SM100-NEXT: [[TMP4:%.*]] = bitcast b8 [[L211]] to i8
+; SM100-NEXT: [[L312:%.*]] = extractelement <32 x b8> [[TMP1]], i32 3
+; SM100-NEXT: [[TMP5:%.*]] = bitcast b8 [[L312]] to i8
+; SM100-NEXT: [[L413:%.*]] = extractelement <32 x b8> [[TMP1]], i32 4
+; SM100-NEXT: [[TMP6:%.*]] = bitcast b8 [[L413]] to i8
+; SM100-NEXT: [[L514:%.*]] = extractelement <32 x b8> [[TMP1]], i32 5
+; SM100-NEXT: [[TMP7:%.*]] = bitcast b8 [[L514]] to i8
+; SM100-NEXT: [[L615:%.*]] = extractelement <32 x b8> [[TMP1]], i32 6
+; SM100-NEXT: [[TMP8:%.*]] = bitcast b8 [[L615]] to i8
+; SM100-NEXT: [[L716:%.*]] = extractelement <32 x b8> [[TMP1]], i32 7
+; SM100-NEXT: [[TMP9:%.*]] = bitcast b8 [[L716]] to i8
+; SM100-NEXT: [[L817:%.*]] = extractelement <32 x b8> [[TMP1]], i32 8
+; SM100-NEXT: [[TMP10:%.*]] = bitcast b8 [[L817]] to i8
+; SM100-NEXT: [[L918:%.*]] = extractelement <32 x b8> [[TMP1]], i32 9
+; SM100-NEXT: [[TMP11:%.*]] = bitcast b8 [[L918]] to i8
+; SM100-NEXT: [[LA19:%.*]] = extractelement <32 x b8> [[TMP1]], i32 10
+; SM100-NEXT: [[TMP12:%.*]] = bitcast b8 [[LA19]] to i8
+; SM100-NEXT: [[LB20:%.*]] = extractelement <32 x b8> [[TMP1]], i32 11
+; SM100-NEXT: [[TMP13:%.*]] = bitcast b8 [[LB20]] to i8
+; SM100-NEXT: [[LC21:%.*]] = extractelement <32 x b8> [[TMP1]], i32 12
+; SM100-NEXT: [[TMP14:%.*]] = bitcast b8 [[LC21]] to i8
+; SM100-NEXT: [[LD22:%.*]] = extractelement <32 x b8> [[TMP1]], i32 13
+; SM100-NEXT: [[TMP15:%.*]] = bitcast b8 [[LD22]] to i8
+; SM100-NEXT: [[LE23:%.*]] = extractelement <32 x b8> [[TMP1]], i32 14
+; SM100-NEXT: [[TMP16:%.*]] = bitcast b8 [[LE23]] to i8
+; SM100-NEXT: [[LF24:%.*]] = extractelement <32 x b8> [[TMP1]], i32 15
+; SM100-NEXT: [[TMP17:%.*]] = bitcast b8 [[LF24]] to i8
+; SM100-NEXT: [[L1025:%.*]] = extractelement <32 x b8> [[TMP1]], i32 16
+; SM100-NEXT: [[TMP18:%.*]] = bitcast b8 [[L1025]] to i8
+; SM100-NEXT: [[L1126:%.*]] = extractelement <32 x b8> [[TMP1]], i32 17
+; SM100-NEXT: [[TMP19:%.*]] = bitcast b8 [[L1126]] to i8
+; SM100-NEXT: [[L1227:%.*]] = extractelement <32 x b8> [[TMP1]], i32 18
+; SM100-NEXT: [[TMP20:%.*]] = bitcast b8 [[L1227]] to i8
+; SM100-NEXT: [[L1328:%.*]] = extractelement <32 x b8> [[TMP1]], i32 19
+; SM100-NEXT: [[TMP21:%.*]] = bitcast b8 [[L1328]] to i8
+; SM100-NEXT: [[L1429:%.*]] = extractelement <32 x b8> [[TMP1]], i32 20
+; SM100-NEXT: [[TMP22:%.*]] = bitcast b8 [[L1429]] to i8
+; SM100-NEXT: [[L1530:%.*]] = extractelement <32 x b8> [[TMP1]], i32 21
+; SM100-NEXT: [[TMP23:%.*]] = bitcast b8 [[L1530]] to i8
+; SM100-NEXT: [[L1631:%.*]] = extractelement <32 x b8> [[TMP1]], i32 22
+; SM100-NEXT: [[TMP24:%.*]] = bitcast b8 [[L1631]] to i8
+; SM100-NEXT: [[L1732:%.*]] = extractelement <32 x b8> [[TMP1]], i32 23
+; SM100-NEXT: [[TMP25:%.*]] = bitcast b8 [[L1732]] to i8
+; SM100-NEXT: [[L1833:%.*]] = extractelement <32 x b8> [[TMP1]], i32 24
+; SM100-NEXT: [[TMP26:%.*]] = bitcast b8 [[L1833]] to i8
+; SM100-NEXT: [[L1934:%.*]] = extractelement <32 x b8> [[TMP1]], i32 25
+; SM100-NEXT: [[TMP27:%.*]] = bitcast b8 [[L1934]] to i8
+; SM100-NEXT: [[L1A35:%.*]] = extractelement <32 x b8> [[TMP1]], i32 26
+; SM100-NEXT: [[TMP28:%.*]] = bitcast b8 [[L1A35]] to i8
+; SM100-NEXT: [[L1B36:%.*]] = extractelement <32 x b8> [[TMP1]], i32 27
+; SM100-NEXT: [[TMP29:%.*]] = bitcast b8 [[L1B36]] to i8
+; SM100-NEXT: [[L1C37:%.*]] = extractelement <32 x b8> [[TMP1]], i32 28
+; SM100-NEXT: [[TMP30:%.*]] = bitcast b8 [[L1C37]] to i8
+; SM100-NEXT: [[L1D38:%.*]] = extractelement <32 x b8> [[TMP1]], i32 29
+; SM100-NEXT: [[TMP31:%.*]] = bitcast b8 [[L1D38]] to i8
+; SM100-NEXT: [[L1E39:%.*]] = extractelement <32 x b8> [[TMP1]], i32 30
+; SM100-NEXT: [[TMP32:%.*]] = bitcast b8 [[L1E39]] to i8
+; SM100-NEXT: [[L1F40:%.*]] = extractelement <32 x b8> [[TMP1]], i32 31
+; SM100-NEXT: [[TMP33:%.*]] = bitcast b8 [[L1F40]] to i8
+; SM100-NEXT: [[TMP34:%.*]] = bitcast i8 [[TMP13]] to b8
+; SM100-NEXT: [[TMP35:%.*]] = insertelement <32 x b8> poison, b8 [[TMP34]], i32 0
+; SM100-NEXT: [[TMP36:%.*]] = bitcast i8 [[TMP12]] to b8
+; SM100-NEXT: [[TMP37:%.*]] = insertelement <32 x b8> [[TMP35]], b8 [[TMP36]], i32 1
+; SM100-NEXT: [[TMP38:%.*]] = bitcast i8 [[TMP11]] to b8
+; SM100-NEXT: [[TMP39:%.*]] = insertelement <32 x b8> [[TMP37]], b8 [[TMP38]], i32 2
+; SM100-NEXT: [[TMP40:%.*]] = bitcast i8 [[TMP10]] to b8
+; SM100-NEXT: [[TMP41:%.*]] = insertelement <32 x b8> [[TMP39]], b8 [[TMP40]], i32 3
+; SM100-NEXT: [[TMP42:%.*]] = bitcast i8 [[TMP9]] to b8
+; SM100-NEXT: [[TMP43:%.*]] = insertelement <32 x b8> [[TMP41]], b8 [[TMP42]], i32 4
+; SM100-NEXT: [[TMP44:%.*]] = bitcast i8 [[TMP8]] to b8
+; SM100-NEXT: [[TMP45:%.*]] = insertelement <32 x b8> [[TMP43]], b8 [[TMP44]], i32 5
+; SM100-NEXT: [[TMP46:%.*]] = bitcast i8 [[TMP7]] to b8
+; SM100-NEXT: [[TMP47:%.*]] = insertelement <32 x b8> [[TMP45]], b8 [[TMP46]], i32 6
+; SM100-NEXT: [[TMP48:%.*]] = bitcast i8 [[TMP6]] to b8
+; SM100-NEXT: [[TMP49:%.*]] = insertelement <32 x b8> [[TMP47]], b8 [[TMP48]], i32 7
+; SM100-NEXT: [[TMP50:%.*]] = bitcast i8 [[TMP5]] to b8
+; SM100-NEXT: [[TMP51:%.*]] = insertelement <32 x b8> [[TMP49]], b8 [[TMP50]], i32 8
+; SM100-NEXT: [[TMP52:%.*]] = bitcast i8 [[TMP4]] to b8
+; SM100-NEXT: [[TMP53:%.*]] = insertelement <32 x b8> [[TMP51]], b8 [[TMP52]], i32 9
+; SM100-NEXT: [[TMP54:%.*]] = bitcast i8 [[TMP3]] to b8
+; SM100-NEXT: [[TMP55:%.*]] = insertelement <32 x b8> [[TMP53]], b8 [[TMP54]], i32 10
+; SM100-NEXT: [[TMP56:%.*]] = bitcast i8 [[TMP2]] to b8
+; SM100-NEXT: [[TMP57:%.*]] = insertelement <32 x b8> [[TMP55]], b8 [[TMP56]], i32 11
+; SM100-NEXT: [[TMP58:%.*]] = bitcast i8 [[TMP17]] to b8
+; SM100-NEXT: [[TMP59:%.*]] = insertelement <32 x b8> [[TMP57]], b8 [[TMP58]], i32 12
+; SM100-NEXT: [[TMP60:%.*]] = bitcast i8 [[TMP16]] to b8
+; SM100-NEXT: [[TMP61:%.*]] = insertelement <32 x b8> [[TMP59]], b8 [[TMP60]], i32 13
+; SM100-NEXT: [[TMP62:%.*]] = bitcast i8 [[TMP15]] to b8
+; SM100-NEXT: [[TMP63:%.*]] = insertelement <32 x b8> [[TMP61]], b8 [[TMP62]], i32 14
+; SM100-NEXT: [[TMP64:%.*]] = bitcast i8 [[TMP14]] to b8
+; SM100-NEXT: [[TMP65:%.*]] = insertelement <32 x b8> [[TMP63]], b8 [[TMP64]], i32 15
+; SM100-NEXT: [[TMP66:%.*]] = bitcast i8 [[TMP29]] to b8
+; SM100-NEXT: [[TMP67:%.*]] = insertelement <32 x b8> [[TMP65]], b8 [[TMP66]], i32 16
+; SM100-NEXT: [[TMP68:%.*]] = bitcast i8 [[TMP28]] to b8
+; SM100-NEXT: [[TMP69:%.*]] = insertelement <32 x b8> [[TMP67]], b8 [[TMP68]], i32 17
+; SM100-NEXT: [[TMP70:%.*]] = bitcast i8 [[TMP27]] to b8
+; SM100-NEXT: [[TMP71:%.*]] = insertelement <32 x b8> [[TMP69]], b8 [[TMP70]], i32 18
+; SM100-NEXT: [[TMP72:%.*]] = bitcast i8 [[TMP26]] to b8
+; SM100-NEXT: [[TMP73:%.*]] = insertelement <32 x b8> [[TMP71]], b8 [[TMP72]], i32 19
+; SM100-NEXT: [[TMP74:%.*]] = bitcast i8 [[TMP25]] to b8
+; SM100-NEXT: [[TMP75:%.*]] = insertelement <32 x b8> [[TMP73]], b8 [[TMP74]], i32 20
+; SM100-NEXT: [[TMP76:%.*]] = bitcast i8 [[TMP24]] to b8
+; SM100-NEXT: [[TMP77:%.*]] = insertelement <32 x b8> [[TMP75]], b8 [[TMP76]], i32 21
+; SM100-NEXT: [[TMP78:%.*]] = bitcast i8 [[TMP23]] to b8
+; SM100-NEXT: [[TMP79:%.*]] = insertelement <32 x b8> [[TMP77]], b8 [[TMP78]], i32 22
+; SM100-NEXT: [[TMP80:%.*]] = bitcast i8 [[TMP22]] to b8
+; SM100-NEXT: [[TMP81:%.*]] = insertelement <32 x b8> [[TMP79]], b8 [[TMP80]], i32 23
+; SM100-NEXT: [[TMP82:%.*]] = bitcast i8 [[TMP21]] to b8
+; SM100-NEXT: [[TMP83:%.*]] = insertelement <32 x b8> [[TMP81]], b8 [[TMP82]], i32 24
+; SM100-NEXT: [[TMP84:%.*]] = bitcast i8 [[TMP20]] to b8
+; SM100-NEXT: [[TMP85:%.*]] = insertelement <32 x b8> [[TMP83]], b8 [[TMP84]], i32 25
+; SM100-NEXT: [[TMP86:%.*]] = bitcast i8 [[TMP19]] to b8
+; SM100-NEXT: [[TMP87:%.*]] = insertelement <32 x b8> [[TMP85]], b8 [[TMP86]], i32 26
+; SM100-NEXT: [[TMP88:%.*]] = bitcast i8 [[TMP18]] to b8
+; SM100-NEXT: [[TMP89:%.*]] = insertelement <32 x b8> [[TMP87]], b8 [[TMP88]], i32 27
+; SM100-NEXT: [[TMP90:%.*]] = bitcast i8 [[TMP33]] to b8
+; SM100-NEXT: [[TMP91:%.*]] = insertelement <32 x b8> [[TMP89]], b8 [[TMP90]], i32 28
+; SM100-NEXT: [[TMP92:%.*]] = bitcast i8 [[TMP32]] to b8
+; SM100-NEXT: [[TMP93:%.*]] = insertelement <32 x b8> [[TMP91]], b8 [[TMP92]], i32 29
+; SM100-NEXT: [[TMP94:%.*]] = bitcast i8 [[TMP31]] to b8
+; SM100-NEXT: [[TMP95:%.*]] = insertelement <32 x b8> [[TMP93]], b8 [[TMP94]], i32 30
+; SM100-NEXT: [[TMP96:%.*]] = bitcast i8 [[TMP30]] to b8
+; SM100-NEXT: [[TMP97:%.*]] = insertelement <32 x b8> [[TMP95]], b8 [[TMP96]], i32 31
+; SM100-NEXT: store <32 x b8> [[TMP97]], ptr addrspace(1) [[PTR0]], align 32
; SM100-NEXT: ret void
;
@@ -270,81 +398,145 @@ define void @int16x16(ptr addrspace(1) %ptr) {
; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
; SM90-NEXT: [[PTR0:%.*]] = getelementptr i16, ptr addrspace(1) [[PTR]], i64 0
; SM90-NEXT: [[PTR8:%.*]] = getelementptr i16, ptr addrspace(1) [[PTR]], i64 8
-; SM90-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[L01:%.*]] = extractelement <8 x i16> [[TMP1]], i32 0
-; SM90-NEXT: [[L12:%.*]] = extractelement <8 x i16> [[TMP1]], i32 1
-; SM90-NEXT: [[L23:%.*]] = extractelement <8 x i16> [[TMP1]], i32 2
-; SM90-NEXT: [[L34:%.*]] = extractelement <8 x i16> [[TMP1]], i32 3
-; SM90-NEXT: [[L45:%.*]] = extractelement <8 x i16> [[TMP1]], i32 4
-; SM90-NEXT: [[L56:%.*]] = extractelement <8 x i16> [[TMP1]], i32 5
-; SM90-NEXT: [[L67:%.*]] = extractelement <8 x i16> [[TMP1]], i32 6
-; SM90-NEXT: [[L78:%.*]] = extractelement <8 x i16> [[TMP1]], i32 7
-; SM90-NEXT: [[TMP2:%.*]] = load <8 x i16>, ptr addrspace(1) [[PTR8]], align 16
-; SM90-NEXT: [[L89:%.*]] = extractelement <8 x i16> [[TMP2]], i32 0
-; SM90-NEXT: [[L910:%.*]] = extractelement <8 x i16> [[TMP2]], i32 1
-; SM90-NEXT: [[LA11:%.*]] = extractelement <8 x i16> [[TMP2]], i32 2
-; SM90-NEXT: [[LB12:%.*]] = extractelement <8 x i16> [[TMP2]], i32 3
-; SM90-NEXT: [[LC13:%.*]] = extractelement <8 x i16> [[TMP2]], i32 4
-; SM90-NEXT: [[LD14:%.*]] = extractelement <8 x i16> [[TMP2]], i32 5
-; SM90-NEXT: [[LE15:%.*]] = extractelement <8 x i16> [[TMP2]], i32 6
-; SM90-NEXT: [[LF16:%.*]] = extractelement <8 x i16> [[TMP2]], i32 7
-; SM90-NEXT: [[TMP3:%.*]] = insertelement <8 x i16> poison, i16 [[LB12]], i32 0
-; SM90-NEXT: [[TMP4:%.*]] = insertelement <8 x i16> [[TMP3]], i16 [[LA11]], i32 1
-; SM90-NEXT: [[TMP5:%.*]] = insertelement <8 x i16> [[TMP4]], i16 [[L910]], i32 2
-; SM90-NEXT: [[TMP6:%.*]] = insertelement <8 x i16> [[TMP5]], i16 [[L89]], i32 3
-; SM90-NEXT: [[TMP7:%.*]] = insertelement <8 x i16> [[TMP6]], i16 [[L78]], i32 4
-; SM90-NEXT: [[TMP8:%.*]] = insertelement <8 x i16> [[TMP7]], i16 [[L67]], i32 5
-; SM90-NEXT: [[TMP9:%.*]] = insertelement <8 x i16> [[TMP8]], i16 [[L56]], i32 6
-; SM90-NEXT: [[TMP10:%.*]] = insertelement <8 x i16> [[TMP9]], i16 [[L45]], i32 7
-; SM90-NEXT: store <8 x i16> [[TMP10]], ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[TMP11:%.*]] = insertelement <8 x i16> poison, i16 [[L34]], i32 0
-; SM90-NEXT: [[TMP12:%.*]] = insertelement <8 x i16> [[TMP11]], i16 [[L23]], i32 1
-; SM90-NEXT: [[TMP13:%.*]] = insertelement <8 x i16> [[TMP12]], i16 [[L12]], i32 2
-; SM90-NEXT: [[TMP14:%.*]] = insertelement <8 x i16> [[TMP13]], i16 [[L01]], i32 3
-; SM90-NEXT: [[TMP15:%.*]] = insertelement <8 x i16> [[TMP14]], i16 [[LF16]], i32 4
-; SM90-NEXT: [[TMP16:%.*]] = insertelement <8 x i16> [[TMP15]], i16 [[LE15]], i32 5
-; SM90-NEXT: [[TMP17:%.*]] = insertelement <8 x i16> [[TMP16]], i16 [[LD14]], i32 6
-; SM90-NEXT: [[TMP18:%.*]] = insertelement <8 x i16> [[TMP17]], i16 [[LC13]], i32 7
-; SM90-NEXT: store <8 x i16> [[TMP18]], ptr addrspace(1) [[PTR8]], align 16
+; SM90-NEXT: [[TMP1:%.*]] = load <8 x b16>, ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[L01:%.*]] = extractelement <8 x b16> [[TMP1]], i32 0
+; SM90-NEXT: [[TMP2:%.*]] = bitcast b16 [[L01]] to i16
+; SM90-NEXT: [[L12:%.*]] = extractelement <8 x b16> [[TMP1]], i32 1
+; SM90-NEXT: [[TMP3:%.*]] = bitcast b16 [[L12]] to i16
+; SM90-NEXT: [[L23:%.*]] = extractelement <8 x b16> [[TMP1]], i32 2
+; SM90-NEXT: [[TMP4:%.*]] = bitcast b16 [[L23]] to i16
+; SM90-NEXT: [[L34:%.*]] = extractelement <8 x b16> [[TMP1]], i32 3
+; SM90-NEXT: [[TMP5:%.*]] = bitcast b16 [[L34]] to i16
+; SM90-NEXT: [[L45:%.*]] = extractelement <8 x b16> [[TMP1]], i32 4
+; SM90-NEXT: [[TMP6:%.*]] = bitcast b16 [[L45]] to i16
+; SM90-NEXT: [[L56:%.*]] = extractelement <8 x b16> [[TMP1]], i32 5
+; SM90-NEXT: [[TMP7:%.*]] = bitcast b16 [[L56]] to i16
+; SM90-NEXT: [[L67:%.*]] = extractelement <8 x b16> [[TMP1]], i32 6
+; SM90-NEXT: [[TMP8:%.*]] = bitcast b16 [[L67]] to i16
+; SM90-NEXT: [[L78:%.*]] = extractelement <8 x b16> [[TMP1]], i32 7
+; SM90-NEXT: [[TMP9:%.*]] = bitcast b16 [[L78]] to i16
+; SM90-NEXT: [[TMP10:%.*]] = load <8 x b16>, ptr addrspace(1) [[PTR8]], align 16
+; SM90-NEXT: [[L89:%.*]] = extractelement <8 x b16> [[TMP10]], i32 0
+; SM90-NEXT: [[TMP11:%.*]] = bitcast b16 [[L89]] to i16
+; SM90-NEXT: [[L910:%.*]] = extractelement <8 x b16> [[TMP10]], i32 1
+; SM90-NEXT: [[TMP12:%.*]] = bitcast b16 [[L910]] to i16
+; SM90-NEXT: [[LA11:%.*]] = extractelement <8 x b16> [[TMP10]], i32 2
+; SM90-NEXT: [[TMP13:%.*]] = bitcast b16 [[LA11]] to i16
+; SM90-NEXT: [[LB12:%.*]] = extractelement <8 x b16> [[TMP10]], i32 3
+; SM90-NEXT: [[TMP14:%.*]] = bitcast b16 [[LB12]] to i16
+; SM90-NEXT: [[LC13:%.*]] = extractelement <8 x b16> [[TMP10]], i32 4
+; SM90-NEXT: [[TMP15:%.*]] = bitcast b16 [[LC13]] to i16
+; SM90-NEXT: [[LD14:%.*]] = extractelement <8 x b16> [[TMP10]], i32 5
+; SM90-NEXT: [[TMP16:%.*]] = bitcast b16 [[LD14]] to i16
+; SM90-NEXT: [[LE15:%.*]] = extractelement <8 x b16> [[TMP10]], i32 6
+; SM90-NEXT: [[TMP17:%.*]] = bitcast b16 [[LE15]] to i16
+; SM90-NEXT: [[LF16:%.*]] = extractelement <8 x b16> [[TMP10]], i32 7
+; SM90-NEXT: [[TMP18:%.*]] = bitcast b16 [[LF16]] to i16
+; SM90-NEXT: [[TMP19:%.*]] = bitcast i16 [[TMP14]] to b16
+; SM90-NEXT: [[TMP20:%.*]] = insertelement <8 x b16> poison, b16 [[TMP19]], i32 0
+; SM90-NEXT: [[TMP21:%.*]] = bitcast i16 [[TMP13]] to b16
+; SM90-NEXT: [[TMP22:%.*]] = insertelement <8 x b16> [[TMP20]], b16 [[TMP21]], i32 1
+; SM90-NEXT: [[TMP23:%.*]] = bitcast i16 [[TMP12]] to b16
+; SM90-NEXT: [[TMP24:%.*]] = insertelement <8 x b16> [[TMP22]], b16 [[TMP23]], i32 2
+; SM90-NEXT: [[TMP25:%.*]] = bitcast i16 [[TMP11]] to b16
+; SM90-NEXT: [[TMP26:%.*]] = insertelement <8 x b16> [[TMP24]], b16 [[TMP25]], i32 3
+; SM90-NEXT: [[TMP27:%.*]] = bitcast i16 [[TMP9]] to b16
+; SM90-NEXT: [[TMP28:%.*]] = insertelement <8 x b16> [[TMP26]], b16 [[TMP27]], i32 4
+; SM90-NEXT: [[TMP29:%.*]] = bitcast i16 [[TMP8]] to b16
+; SM90-NEXT: [[TMP30:%.*]] = insertelement <8 x b16> [[TMP28]], b16 [[TMP29]], i32 5
+; SM90-NEXT: [[TMP31:%.*]] = bitcast i16 [[TMP7]] to b16
+; SM90-NEXT: [[TMP32:%.*]] = insertelement <8 x b16> [[TMP30]], b16 [[TMP31]], i32 6
+; SM90-NEXT: [[TMP33:%.*]] = bitcast i16 [[TMP6]] to b16
+; SM90-NEXT: [[TMP34:%.*]] = insertelement <8 x b16> [[TMP32]], b16 [[TMP33]], i32 7
+; SM90-NEXT: store <8 x b16> [[TMP34]], ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[TMP35:%.*]] = bitcast i16 [[TMP5]] to b16
+; SM90-NEXT: [[TMP36:%.*]] = insertelement <8 x b16> poison, b16 [[TMP35]], i32 0
+; SM90-NEXT: [[TMP37:%.*]] = bitcast i16 [[TMP4]] to b16
+; SM90-NEXT: [[TMP38:%.*]] = insertelement <8 x b16> [[TMP36]], b16 [[TMP37]], i32 1
+; SM90-NEXT: [[TMP39:%.*]] = bitcast i16 [[TMP3]] to b16
+; SM90-NEXT: [[TMP40:%.*]] = insertelement <8 x b16> [[TMP38]], b16 [[TMP39]], i32 2
+; SM90-NEXT: [[TMP41:%.*]] = bitcast i16 [[TMP2]] to b16
+; SM90-NEXT: [[TMP42:%.*]] = insertelement <8 x b16> [[TMP40]], b16 [[TMP41]], i32 3
+; SM90-NEXT: [[TMP43:%.*]] = bitcast i16 [[TMP18]] to b16
+; SM90-NEXT: [[TMP44:%.*]] = insertelement <8 x b16> [[TMP42]], b16 [[TMP43]], i32 4
+; SM90-NEXT: [[TMP45:%.*]] = bitcast i16 [[TMP17]] to b16
+; SM90-NEXT: [[TMP46:%.*]] = insertelement <8 x b16> [[TMP44]], b16 [[TMP45]], i32 5
+; SM90-NEXT: [[TMP47:%.*]] = bitcast i16 [[TMP16]] to b16
+; SM90-NEXT: [[TMP48:%.*]] = insertelement <8 x b16> [[TMP46]], b16 [[TMP47]], i32 6
+; SM90-NEXT: [[TMP49:%.*]] = bitcast i16 [[TMP15]] to b16
+; SM90-NEXT: [[TMP50:%.*]] = insertelement <8 x b16> [[TMP48]], b16 [[TMP49]], i32 7
+; SM90-NEXT: store <8 x b16> [[TMP50]], ptr addrspace(1) [[PTR8]], align 16
; SM90-NEXT: ret void
;
; SM100-LABEL: define void @int16x16(
; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
; SM100-NEXT: [[PTR0:%.*]] = getelementptr i16, ptr addrspace(1) [[PTR]], i64 0
-; SM100-NEXT: [[TMP1:%.*]] = load <16 x i16>, ptr addrspace(1) [[PTR0]], align 32
-; SM100-NEXT: [[L01:%.*]] = extractelement <16 x i16> [[TMP1]], i32 0
-; SM100-NEXT: [[L12:%.*]] = extractelement <16 x i16> [[TMP1]], i32 1
-; SM100-NEXT: [[L23:%.*]] = extractelement <16 x i16> [[TMP1]], i32 2
-; SM100-NEXT: [[L34:%.*]] = extractelement <16 x i16> [[TMP1]], i32 3
-; SM100-NEXT: [[L45:%.*]] = extractelement <16 x i16> [[TMP1]], i32 4
-; SM100-NEXT: [[L56:%.*]] = extractelement <16 x i16> [[TMP1]], i32 5
-; SM100-NEXT: [[L67:%.*]] = extractelement <16 x i16> [[TMP1]], i32 6
-; SM100-NEXT: [[L78:%.*]] = extractelement <16 x i16> [[TMP1]], i32 7
-; SM100-NEXT: [[L89:%.*]] = extractelement <16 x i16> [[TMP1]], i32 8
-; SM100-NEXT: [[L910:%.*]] = extractelement <16 x i16> [[TMP1]], i32 9
-; SM100-NEXT: [[LA11:%.*]] = extractelement <16 x i16> [[TMP1]], i32 10
-; SM100-NEXT: [[LB12:%.*]] = extractelement <16 x i16> [[TMP1]], i32 11
-; SM100-NEXT: [[LC13:%.*]] = extractelement <16 x i16> [[TMP1]], i32 12
-; SM100-NEXT: [[LD14:%.*]] = extractelement <16 x i16> [[TMP1]], i32 13
-; SM100-NEXT: [[LE15:%.*]] = extractelement <16 x i16> [[TMP1]], i32 14
-; SM100-NEXT: [[LF16:%.*]] = extractelement <16 x i16> [[TMP1]], i32 15
-; SM100-NEXT: [[TMP2:%.*]] = insertelement <16 x i16> poison, i16 [[LB12]], i32 0
-; SM100-NEXT: [[TMP3:%.*]] = insertelement <16 x i16> [[TMP2]], i16 [[LA11]], i32 1
-; SM100-NEXT: [[TMP4:%.*]] = insertelement <16 x i16> [[TMP3]], i16 [[L910]], i32 2
-; SM100-NEXT: [[TMP5:%.*]] = insertelement <16 x i16> [[TMP4]], i16 [[L89]], i32 3
-; SM100-NEXT: [[TMP6:%.*]] = insertelement <16 x i16> [[TMP5]], i16 [[L78]], i32 4
-; SM100-NEXT: [[TMP7:%.*]] = insertelement <16 x i16> [[TMP6]], i16 [[L67]], i32 5
-; SM100-NEXT: [[TMP8:%.*]] = insertelement <16 x i16> [[TMP7]], i16 [[L56]], i32 6
-; SM100-NEXT: [[TMP9:%.*]] = insertelement <16 x i16> [[TMP8]], i16 [[L45]], i32 7
-; SM100-NEXT: [[TMP10:%.*]] = insertelement <16 x i16> [[TMP9]], i16 [[L34]], i32 8
-; SM100-NEXT: [[TMP11:%.*]] = insertelement <16 x i16> [[TMP10]], i16 [[L23]], i32 9
-; SM100-NEXT: [[TMP12:%.*]] = insertelement <16 x i16> [[TMP11]], i16 [[L12]], i32 10
-; SM100-NEXT: [[TMP13:%.*]] = insertelement <16 x i16> [[TMP12]], i16 [[L01]], i32 11
-; SM100-NEXT: [[TMP14:%.*]] = insertelement <16 x i16> [[TMP13]], i16 [[LF16]], i32 12
-; SM100-NEXT: [[TMP15:%.*]] = insertelement <16 x i16> [[TMP14]], i16 [[LE15]], i32 13
-; SM100-NEXT: [[TMP16:%.*]] = insertelement <16 x i16> [[TMP15]], i16 [[LD14]], i32 14
-; SM100-NEXT: [[TMP17:%.*]] = insertelement <16 x i16> [[TMP16]], i16 [[LC13]], i32 15
-; SM100-NEXT: store <16 x i16> [[TMP17]], ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[TMP1:%.*]] = load <16 x b16>, ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[L01:%.*]] = extractelement <16 x b16> [[TMP1]], i32 0
+; SM100-NEXT: [[TMP2:%.*]] = bitcast b16 [[L01]] to i16
+; SM100-NEXT: [[L12:%.*]] = extractelement <16 x b16> [[TMP1]], i32 1
+; SM100-NEXT: [[TMP3:%.*]] = bitcast b16 [[L12]] to i16
+; SM100-NEXT: [[L23:%.*]] = extractelement <16 x b16> [[TMP1]], i32 2
+; SM100-NEXT: [[TMP4:%.*]] = bitcast b16 [[L23]] to i16
+; SM100-NEXT: [[L34:%.*]] = extractelement <16 x b16> [[TMP1]], i32 3
+; SM100-NEXT: [[TMP5:%.*]] = bitcast b16 [[L34]] to i16
+; SM100-NEXT: [[L45:%.*]] = extractelement <16 x b16> [[TMP1]], i32 4
+; SM100-NEXT: [[TMP6:%.*]] = bitcast b16 [[L45]] to i16
+; SM100-NEXT: [[L56:%.*]] = extractelement <16 x b16> [[TMP1]], i32 5
+; SM100-NEXT: [[TMP7:%.*]] = bitcast b16 [[L56]] to i16
+; SM100-NEXT: [[L67:%.*]] = extractelement <16 x b16> [[TMP1]], i32 6
+; SM100-NEXT: [[TMP8:%.*]] = bitcast b16 [[L67]] to i16
+; SM100-NEXT: [[L78:%.*]] = extractelement <16 x b16> [[TMP1]], i32 7
+; SM100-NEXT: [[TMP9:%.*]] = bitcast b16 [[L78]] to i16
+; SM100-NEXT: [[L89:%.*]] = extractelement <16 x b16> [[TMP1]], i32 8
+; SM100-NEXT: [[TMP10:%.*]] = bitcast b16 [[L89]] to i16
+; SM100-NEXT: [[L910:%.*]] = extractelement <16 x b16> [[TMP1]], i32 9
+; SM100-NEXT: [[TMP11:%.*]] = bitcast b16 [[L910]] to i16
+; SM100-NEXT: [[LA11:%.*]] = extractelement <16 x b16> [[TMP1]], i32 10
+; SM100-NEXT: [[TMP12:%.*]] = bitcast b16 [[LA11]] to i16
+; SM100-NEXT: [[LB12:%.*]] = extractelement <16 x b16> [[TMP1]], i32 11
+; SM100-NEXT: [[TMP13:%.*]] = bitcast b16 [[LB12]] to i16
+; SM100-NEXT: [[LC13:%.*]] = extractelement <16 x b16> [[TMP1]], i32 12
+; SM100-NEXT: [[TMP14:%.*]] = bitcast b16 [[LC13]] to i16
+; SM100-NEXT: [[LD14:%.*]] = extractelement <16 x b16> [[TMP1]], i32 13
+; SM100-NEXT: [[TMP15:%.*]] = bitcast b16 [[LD14]] to i16
+; SM100-NEXT: [[LE15:%.*]] = extractelement <16 x b16> [[TMP1]], i32 14
+; SM100-NEXT: [[TMP16:%.*]] = bitcast b16 [[LE15]] to i16
+; SM100-NEXT: [[LF16:%.*]] = extractelement <16 x b16> [[TMP1]], i32 15
+; SM100-NEXT: [[TMP17:%.*]] = bitcast b16 [[LF16]] to i16
+; SM100-NEXT: [[TMP18:%.*]] = bitcast i16 [[TMP13]] to b16
+; SM100-NEXT: [[TMP19:%.*]] = insertelement <16 x b16> poison, b16 [[TMP18]], i32 0
+; SM100-NEXT: [[TMP20:%.*]] = bitcast i16 [[TMP12]] to b16
+; SM100-NEXT: [[TMP21:%.*]] = insertelement <16 x b16> [[TMP19]], b16 [[TMP20]], i32 1
+; SM100-NEXT: [[TMP22:%.*]] = bitcast i16 [[TMP11]] to b16
+; SM100-NEXT: [[TMP23:%.*]] = insertelement <16 x b16> [[TMP21]], b16 [[TMP22]], i32 2
+; SM100-NEXT: [[TMP24:%.*]] = bitcast i16 [[TMP10]] to b16
+; SM100-NEXT: [[TMP25:%.*]] = insertelement <16 x b16> [[TMP23]], b16 [[TMP24]], i32 3
+; SM100-NEXT: [[TMP26:%.*]] = bitcast i16 [[TMP9]] to b16
+; SM100-NEXT: [[TMP27:%.*]] = insertelement <16 x b16> [[TMP25]], b16 [[TMP26]], i32 4
+; SM100-NEXT: [[TMP28:%.*]] = bitcast i16 [[TMP8]] to b16
+; SM100-NEXT: [[TMP29:%.*]] = insertelement <16 x b16> [[TMP27]], b16 [[TMP28]], i32 5
+; SM100-NEXT: [[TMP30:%.*]] = bitcast i16 [[TMP7]] to b16
+; SM100-NEXT: [[TMP31:%.*]] = insertelement <16 x b16> [[TMP29]], b16 [[TMP30]], i32 6
+; SM100-NEXT: [[TMP32:%.*]] = bitcast i16 [[TMP6]] to b16
+; SM100-NEXT: [[TMP33:%.*]] = insertelement <16 x b16> [[TMP31]], b16 [[TMP32]], i32 7
+; SM100-NEXT: [[TMP34:%.*]] = bitcast i16 [[TMP5]] to b16
+; SM100-NEXT: [[TMP35:%.*]] = insertelement <16 x b16> [[TMP33]], b16 [[TMP34]], i32 8
+; SM100-NEXT: [[TMP36:%.*]] = bitcast i16 [[TMP4]] to b16
+; SM100-NEXT: [[TMP37:%.*]] = insertelement <16 x b16> [[TMP35]], b16 [[TMP36]], i32 9
+; SM100-NEXT: [[TMP38:%.*]] = bitcast i16 [[TMP3]] to b16
+; SM100-NEXT: [[TMP39:%.*]] = insertelement <16 x b16> [[TMP37]], b16 [[TMP38]], i32 10
+; SM100-NEXT: [[TMP40:%.*]] = bitcast i16 [[TMP2]] to b16
+; SM100-NEXT: [[TMP41:%.*]] = insertelement <16 x b16> [[TMP39]], b16 [[TMP40]], i32 11
+; SM100-NEXT: [[TMP42:%.*]] = bitcast i16 [[TMP17]] to b16
+; SM100-NEXT: [[TMP43:%.*]] = insertelement <16 x b16> [[TMP41]], b16 [[TMP42]], i32 12
+; SM100-NEXT: [[TMP44:%.*]] = bitcast i16 [[TMP16]] to b16
+; SM100-NEXT: [[TMP45:%.*]] = insertelement <16 x b16> [[TMP43]], b16 [[TMP44]], i32 13
+; SM100-NEXT: [[TMP46:%.*]] = bitcast i16 [[TMP15]] to b16
+; SM100-NEXT: [[TMP47:%.*]] = insertelement <16 x b16> [[TMP45]], b16 [[TMP46]], i32 14
+; SM100-NEXT: [[TMP48:%.*]] = bitcast i16 [[TMP14]] to b16
+; SM100-NEXT: [[TMP49:%.*]] = insertelement <16 x b16> [[TMP47]], b16 [[TMP48]], i32 15
+; SM100-NEXT: store <16 x b16> [[TMP49]], ptr addrspace(1) [[PTR0]], align 32
; SM100-NEXT: ret void
;
@@ -407,49 +599,81 @@ define void @int32x8(ptr addrspace(1) %ptr) {
; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
; SM90-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 0
; SM90-NEXT: [[PTR4:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 4
-; SM90-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[L01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; SM90-NEXT: [[L12:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; SM90-NEXT: [[L23:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; SM90-NEXT: [[L34:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-; SM90-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr addrspace(1) [[PTR4]], align 16
-; SM90-NEXT: [[L45:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
-; SM90-NEXT: [[L56:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
-; SM90-NEXT: [[L67:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
-; SM90-NEXT: [[L78:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
-; SM90-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[L78]], i32 0
-; SM90-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[L67]], i32 1
-; SM90-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[L56]], i32 2
-; SM90-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[L45]], i32 3
-; SM90-NEXT: store <4 x i32> [[TMP6]], ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[L34]], i32 0
-; SM90-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[L23]], i32 1
-; SM90-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[L12]], i32 2
-; SM90-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[L01]], i32 3
-; SM90-NEXT: store <4 x i32> [[TMP10]], ptr addrspace(1) [[PTR4]], align 16
+; SM90-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[L01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; SM90-NEXT: [[TMP2:%.*]] = bitcast b32 [[L01]] to i32
+; SM90-NEXT: [[L12:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; SM90-NEXT: [[TMP3:%.*]] = bitcast b32 [[L12]] to i32
+; SM90-NEXT: [[L23:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; SM90-NEXT: [[TMP4:%.*]] = bitcast b32 [[L23]] to i32
+; SM90-NEXT: [[L34:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; SM90-NEXT: [[TMP5:%.*]] = bitcast b32 [[L34]] to i32
+; SM90-NEXT: [[TMP6:%.*]] = load <4 x b32>, ptr addrspace(1) [[PTR4]], align 16
+; SM90-NEXT: [[L45:%.*]] = extractelement <4 x b32> [[TMP6]], i32 0
+; SM90-NEXT: [[TMP7:%.*]] = bitcast b32 [[L45]] to i32
+; SM90-NEXT: [[L56:%.*]] = extractelement <4 x b32> [[TMP6]], i32 1
+; SM90-NEXT: [[TMP8:%.*]] = bitcast b32 [[L56]] to i32
+; SM90-NEXT: [[L67:%.*]] = extractelement <4 x b32> [[TMP6]], i32 2
+; SM90-NEXT: [[TMP9:%.*]] = bitcast b32 [[L67]] to i32
+; SM90-NEXT: [[L78:%.*]] = extractelement <4 x b32> [[TMP6]], i32 3
+; SM90-NEXT: [[TMP10:%.*]] = bitcast b32 [[L78]] to i32
+; SM90-NEXT: [[TMP11:%.*]] = bitcast i32 [[TMP10]] to b32
+; SM90-NEXT: [[TMP12:%.*]] = insertelement <4 x b32> poison, b32 [[TMP11]], i32 0
+; SM90-NEXT: [[TMP13:%.*]] = bitcast i32 [[TMP9]] to b32
+; SM90-NEXT: [[TMP14:%.*]] = insertelement <4 x b32> [[TMP12]], b32 [[TMP13]], i32 1
+; SM90-NEXT: [[TMP15:%.*]] = bitcast i32 [[TMP8]] to b32
+; SM90-NEXT: [[TMP16:%.*]] = insertelement <4 x b32> [[TMP14]], b32 [[TMP15]], i32 2
+; SM90-NEXT: [[TMP17:%.*]] = bitcast i32 [[TMP7]] to b32
+; SM90-NEXT: [[TMP18:%.*]] = insertelement <4 x b32> [[TMP16]], b32 [[TMP17]], i32 3
+; SM90-NEXT: store <4 x b32> [[TMP18]], ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[TMP19:%.*]] = bitcast i32 [[TMP5]] to b32
+; SM90-NEXT: [[TMP20:%.*]] = insertelement <4 x b32> poison, b32 [[TMP19]], i32 0
+; SM90-NEXT: [[TMP21:%.*]] = bitcast i32 [[TMP4]] to b32
+; SM90-NEXT: [[TMP22:%.*]] = insertelement <4 x b32> [[TMP20]], b32 [[TMP21]], i32 1
+; SM90-NEXT: [[TMP23:%.*]] = bitcast i32 [[TMP3]] to b32
+; SM90-NEXT: [[TMP24:%.*]] = insertelement <4 x b32> [[TMP22]], b32 [[TMP23]], i32 2
+; SM90-NEXT: [[TMP25:%.*]] = bitcast i32 [[TMP2]] to b32
+; SM90-NEXT: [[TMP26:%.*]] = insertelement <4 x b32> [[TMP24]], b32 [[TMP25]], i32 3
+; SM90-NEXT: store <4 x b32> [[TMP26]], ptr addrspace(1) [[PTR4]], align 16
; SM90-NEXT: ret void
;
; SM100-LABEL: define void @int32x8(
; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
; SM100-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr addrspace(1) [[PTR]], i64 0
-; SM100-NEXT: [[TMP1:%.*]] = load <8 x i32>, ptr addrspace(1) [[PTR0]], align 32
-; SM100-NEXT: [[L01:%.*]] = extractelement <8 x i32> [[TMP1]], i32 0
-; SM100-NEXT: [[L12:%.*]] = extractelement <8 x i32> [[TMP1]], i32 1
-; SM100-NEXT: [[L23:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2
-; SM100-NEXT: [[L34:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3
-; SM100-NEXT: [[L45:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4
-; SM100-NEXT: [[L56:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5
-; SM100-NEXT: [[L67:%.*]] = extractelement <8 x i32> [[TMP1]], i32 6
-; SM100-NEXT: [[L78:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7
-; SM100-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[L78]], i32 0
-; SM100-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[L67]], i32 1
-; SM100-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[L56]], i32 2
-; SM100-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[L45]], i32 3
-; SM100-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[L34]], i32 4
-; SM100-NEXT: [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 [[L23]], i32 5
-; SM100-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 [[L12]], i32 6
-; SM100-NEXT: [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 [[L01]], i32 7
-; SM100-NEXT: store <8 x i32> [[TMP9]], ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[TMP1:%.*]] = load <8 x b32>, ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[L01:%.*]] = extractelement <8 x b32> [[TMP1]], i32 0
+; SM100-NEXT: [[TMP2:%.*]] = bitcast b32 [[L01]] to i32
+; SM100-NEXT: [[L12:%.*]] = extractelement <8 x b32> [[TMP1]], i32 1
+; SM100-NEXT: [[TMP3:%.*]] = bitcast b32 [[L12]] to i32
+; SM100-NEXT: [[L23:%.*]] = extractelement <8 x b32> [[TMP1]], i32 2
+; SM100-NEXT: [[TMP4:%.*]] = bitcast b32 [[L23]] to i32
+; SM100-NEXT: [[L34:%.*]] = extractelement <8 x b32> [[TMP1]], i32 3
+; SM100-NEXT: [[TMP5:%.*]] = bitcast b32 [[L34]] to i32
+; SM100-NEXT: [[L45:%.*]] = extractelement <8 x b32> [[TMP1]], i32 4
+; SM100-NEXT: [[TMP6:%.*]] = bitcast b32 [[L45]] to i32
+; SM100-NEXT: [[L56:%.*]] = extractelement <8 x b32> [[TMP1]], i32 5
+; SM100-NEXT: [[TMP7:%.*]] = bitcast b32 [[L56]] to i32
+; SM100-NEXT: [[L67:%.*]] = extractelement <8 x b32> [[TMP1]], i32 6
+; SM100-NEXT: [[TMP8:%.*]] = bitcast b32 [[L67]] to i32
+; SM100-NEXT: [[L78:%.*]] = extractelement <8 x b32> [[TMP1]], i32 7
+; SM100-NEXT: [[TMP9:%.*]] = bitcast b32 [[L78]] to i32
+; SM100-NEXT: [[TMP10:%.*]] = bitcast i32 [[TMP9]] to b32
+; SM100-NEXT: [[TMP11:%.*]] = insertelement <8 x b32> poison, b32 [[TMP10]], i32 0
+; SM100-NEXT: [[TMP12:%.*]] = bitcast i32 [[TMP8]] to b32
+; SM100-NEXT: [[TMP13:%.*]] = insertelement <8 x b32> [[TMP11]], b32 [[TMP12]], i32 1
+; SM100-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP7]] to b32
+; SM100-NEXT: [[TMP15:%.*]] = insertelement <8 x b32> [[TMP13]], b32 [[TMP14]], i32 2
+; SM100-NEXT: [[TMP16:%.*]] = bitcast i32 [[TMP6]] to b32
+; SM100-NEXT: [[TMP17:%.*]] = insertelement <8 x b32> [[TMP15]], b32 [[TMP16]], i32 3
+; SM100-NEXT: [[TMP18:%.*]] = bitcast i32 [[TMP5]] to b32
+; SM100-NEXT: [[TMP19:%.*]] = insertelement <8 x b32> [[TMP17]], b32 [[TMP18]], i32 4
+; SM100-NEXT: [[TMP20:%.*]] = bitcast i32 [[TMP4]] to b32
+; SM100-NEXT: [[TMP21:%.*]] = insertelement <8 x b32> [[TMP19]], b32 [[TMP20]], i32 5
+; SM100-NEXT: [[TMP22:%.*]] = bitcast i32 [[TMP3]] to b32
+; SM100-NEXT: [[TMP23:%.*]] = insertelement <8 x b32> [[TMP21]], b32 [[TMP22]], i32 6
+; SM100-NEXT: [[TMP24:%.*]] = bitcast i32 [[TMP2]] to b32
+; SM100-NEXT: [[TMP25:%.*]] = insertelement <8 x b32> [[TMP23]], b32 [[TMP24]], i32 7
+; SM100-NEXT: store <8 x b32> [[TMP25]], ptr addrspace(1) [[PTR0]], align 32
; SM100-NEXT: ret void
;
@@ -488,33 +712,49 @@ define void @int64x4(ptr addrspace(1) %ptr) {
; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
; SM90-NEXT: [[PTR0:%.*]] = getelementptr i64, ptr addrspace(1) [[PTR]], i64 0
; SM90-NEXT: [[PTR2:%.*]] = getelementptr i64, ptr addrspace(1) [[PTR]], i64 2
-; SM90-NEXT: [[TMP1:%.*]] = load <2 x i64>, ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[L01:%.*]] = extractelement <2 x i64> [[TMP1]], i32 0
-; SM90-NEXT: [[L12:%.*]] = extractelement <2 x i64> [[TMP1]], i32 1
-; SM90-NEXT: [[TMP2:%.*]] = load <2 x i64>, ptr addrspace(1) [[PTR2]], align 16
-; SM90-NEXT: [[L23:%.*]] = extractelement <2 x i64> [[TMP2]], i32 0
-; SM90-NEXT: [[L34:%.*]] = extractelement <2 x i64> [[TMP2]], i32 1
-; SM90-NEXT: [[TMP3:%.*]] = insertelement <2 x i64> poison, i64 [[L34]], i32 0
-; SM90-NEXT: [[TMP4:%.*]] = insertelement <2 x i64> [[TMP3]], i64 [[L23]], i32 1
-; SM90-NEXT: store <2 x i64> [[TMP4]], ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[TMP5:%.*]] = insertelement <2 x i64> poison, i64 [[L12]], i32 0
-; SM90-NEXT: [[TMP6:%.*]] = insertelement <2 x i64> [[TMP5]], i64 [[L01]], i32 1
-; SM90-NEXT: store <2 x i64> [[TMP6]], ptr addrspace(1) [[PTR2]], align 16
+; SM90-NEXT: [[TMP1:%.*]] = load <2 x b64>, ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[L01:%.*]] = extractelement <2 x b64> [[TMP1]], i32 0
+; SM90-NEXT: [[TMP2:%.*]] = bitcast b64 [[L01]] to i64
+; SM90-NEXT: [[L12:%.*]] = extractelement <2 x b64> [[TMP1]], i32 1
+; SM90-NEXT: [[TMP3:%.*]] = bitcast b64 [[L12]] to i64
+; SM90-NEXT: [[TMP4:%.*]] = load <2 x b64>, ptr addrspace(1) [[PTR2]], align 16
+; SM90-NEXT: [[L23:%.*]] = extractelement <2 x b64> [[TMP4]], i32 0
+; SM90-NEXT: [[TMP5:%.*]] = bitcast b64 [[L23]] to i64
+; SM90-NEXT: [[L34:%.*]] = extractelement <2 x b64> [[TMP4]], i32 1
+; SM90-NEXT: [[TMP6:%.*]] = bitcast b64 [[L34]] to i64
+; SM90-NEXT: [[TMP7:%.*]] = bitcast i64 [[TMP6]] to b64
+; SM90-NEXT: [[TMP8:%.*]] = insertelement <2 x b64> poison, b64 [[TMP7]], i32 0
+; SM90-NEXT: [[TMP9:%.*]] = bitcast i64 [[TMP5]] to b64
+; SM90-NEXT: [[TMP10:%.*]] = insertelement <2 x b64> [[TMP8]], b64 [[TMP9]], i32 1
+; SM90-NEXT: store <2 x b64> [[TMP10]], ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[TMP11:%.*]] = bitcast i64 [[TMP3]] to b64
+; SM90-NEXT: [[TMP12:%.*]] = insertelement <2 x b64> poison, b64 [[TMP11]], i32 0
+; SM90-NEXT: [[TMP13:%.*]] = bitcast i64 [[TMP2]] to b64
+; SM90-NEXT: [[TMP14:%.*]] = insertelement <2 x b64> [[TMP12]], b64 [[TMP13]], i32 1
+; SM90-NEXT: store <2 x b64> [[TMP14]], ptr addrspace(1) [[PTR2]], align 16
; SM90-NEXT: ret void
;
; SM100-LABEL: define void @int64x4(
; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
; SM100-NEXT: [[PTR0:%.*]] = getelementptr i64, ptr addrspace(1) [[PTR]], i64 0
-; SM100-NEXT: [[TMP1:%.*]] = load <4 x i64>, ptr addrspace(1) [[PTR0]], align 32
-; SM100-NEXT: [[L01:%.*]] = extractelement <4 x i64> [[TMP1]], i32 0
-; SM100-NEXT: [[L12:%.*]] = extractelement <4 x i64> [[TMP1]], i32 1
-; SM100-NEXT: [[L23:%.*]] = extractelement <4 x i64> [[TMP1]], i32 2
-; SM100-NEXT: [[L34:%.*]] = extractelement <4 x i64> [[TMP1]], i32 3
-; SM100-NEXT: [[TMP2:%.*]] = insertelement <4 x i64> poison, i64 [[L34]], i32 0
-; SM100-NEXT: [[TMP3:%.*]] = insertelement <4 x i64> [[TMP2]], i64 [[L23]], i32 1
-; SM100-NEXT: [[TMP4:%.*]] = insertelement <4 x i64> [[TMP3]], i64 [[L12]], i32 2
-; SM100-NEXT: [[TMP5:%.*]] = insertelement <4 x i64> [[TMP4]], i64 [[L01]], i32 3
-; SM100-NEXT: store <4 x i64> [[TMP5]], ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[TMP1:%.*]] = load <4 x b64>, ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[L01:%.*]] = extractelement <4 x b64> [[TMP1]], i32 0
+; SM100-NEXT: [[TMP2:%.*]] = bitcast b64 [[L01]] to i64
+; SM100-NEXT: [[L12:%.*]] = extractelement <4 x b64> [[TMP1]], i32 1
+; SM100-NEXT: [[TMP3:%.*]] = bitcast b64 [[L12]] to i64
+; SM100-NEXT: [[L23:%.*]] = extractelement <4 x b64> [[TMP1]], i32 2
+; SM100-NEXT: [[TMP4:%.*]] = bitcast b64 [[L23]] to i64
+; SM100-NEXT: [[L34:%.*]] = extractelement <4 x b64> [[TMP1]], i32 3
+; SM100-NEXT: [[TMP5:%.*]] = bitcast b64 [[L34]] to i64
+; SM100-NEXT: [[TMP6:%.*]] = bitcast i64 [[TMP5]] to b64
+; SM100-NEXT: [[TMP7:%.*]] = insertelement <4 x b64> poison, b64 [[TMP6]], i32 0
+; SM100-NEXT: [[TMP8:%.*]] = bitcast i64 [[TMP4]] to b64
+; SM100-NEXT: [[TMP9:%.*]] = insertelement <4 x b64> [[TMP7]], b64 [[TMP8]], i32 1
+; SM100-NEXT: [[TMP10:%.*]] = bitcast i64 [[TMP3]] to b64
+; SM100-NEXT: [[TMP11:%.*]] = insertelement <4 x b64> [[TMP9]], b64 [[TMP10]], i32 2
+; SM100-NEXT: [[TMP12:%.*]] = bitcast i64 [[TMP2]] to b64
+; SM100-NEXT: [[TMP13:%.*]] = insertelement <4 x b64> [[TMP11]], b64 [[TMP12]], i32 3
+; SM100-NEXT: store <4 x b64> [[TMP13]], ptr addrspace(1) [[PTR0]], align 32
; SM100-NEXT: ret void
;
@@ -541,49 +781,81 @@ define void @float32x8(ptr addrspace(1) %ptr) {
; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
; SM90-NEXT: [[PTR0:%.*]] = getelementptr float, ptr addrspace(1) [[PTR]], i64 0
; SM90-NEXT: [[PTR4:%.*]] = getelementptr float, ptr addrspace(1) [[PTR]], i64 4
-; SM90-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[L01:%.*]] = extractelement <4 x float> [[TMP1]], i32 0
-; SM90-NEXT: [[L12:%.*]] = extractelement <4 x float> [[TMP1]], i32 1
-; SM90-NEXT: [[L23:%.*]] = extractelement <4 x float> [[TMP1]], i32 2
-; SM90-NEXT: [[L34:%.*]] = extractelement <4 x float> [[TMP1]], i32 3
-; SM90-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr addrspace(1) [[PTR4]], align 16
-; SM90-NEXT: [[L45:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; SM90-NEXT: [[L56:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
-; SM90-NEXT: [[L67:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
-; SM90-NEXT: [[L78:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
-; SM90-NEXT: [[TMP3:%.*]] = insertelement <4 x float> poison, float [[L78]], i32 0
-; SM90-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[L67]], i32 1
-; SM90-NEXT: [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[L56]], i32 2
-; SM90-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[L45]], i32 3
-; SM90-NEXT: store <4 x float> [[TMP6]], ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[TMP7:%.*]] = insertelement <4 x float> poison, float [[L34]], i32 0
-; SM90-NEXT: [[TMP8:%.*]] = insertelement <4 x float> [[TMP7]], float [[L23]], i32 1
-; SM90-NEXT: [[TMP9:%.*]] = insertelement <4 x float> [[TMP8]], float [[L12]], i32 2
-; SM90-NEXT: [[TMP10:%.*]] = insertelement <4 x float> [[TMP9]], float [[L01]], i32 3
-; SM90-NEXT: store <4 x float> [[TMP10]], ptr addrspace(1) [[PTR4]], align 16
+; SM90-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[L01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; SM90-NEXT: [[TMP2:%.*]] = bitcast b32 [[L01]] to float
+; SM90-NEXT: [[L12:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; SM90-NEXT: [[TMP3:%.*]] = bitcast b32 [[L12]] to float
+; SM90-NEXT: [[L23:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; SM90-NEXT: [[TMP4:%.*]] = bitcast b32 [[L23]] to float
+; SM90-NEXT: [[L34:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; SM90-NEXT: [[TMP5:%.*]] = bitcast b32 [[L34]] to float
+; SM90-NEXT: [[TMP6:%.*]] = load <4 x b32>, ptr addrspace(1) [[PTR4]], align 16
+; SM90-NEXT: [[L45:%.*]] = extractelement <4 x b32> [[TMP6]], i32 0
+; SM90-NEXT: [[TMP7:%.*]] = bitcast b32 [[L45]] to float
+; SM90-NEXT: [[L56:%.*]] = extractelement <4 x b32> [[TMP6]], i32 1
+; SM90-NEXT: [[TMP8:%.*]] = bitcast b32 [[L56]] to float
+; SM90-NEXT: [[L67:%.*]] = extractelement <4 x b32> [[TMP6]], i32 2
+; SM90-NEXT: [[TMP9:%.*]] = bitcast b32 [[L67]] to float
+; SM90-NEXT: [[L78:%.*]] = extractelement <4 x b32> [[TMP6]], i32 3
+; SM90-NEXT: [[TMP10:%.*]] = bitcast b32 [[L78]] to float
+; SM90-NEXT: [[TMP11:%.*]] = bitcast float [[TMP10]] to b32
+; SM90-NEXT: [[TMP12:%.*]] = insertelement <4 x b32> poison, b32 [[TMP11]], i32 0
+; SM90-NEXT: [[TMP13:%.*]] = bitcast float [[TMP9]] to b32
+; SM90-NEXT: [[TMP14:%.*]] = insertelement <4 x b32> [[TMP12]], b32 [[TMP13]], i32 1
+; SM90-NEXT: [[TMP15:%.*]] = bitcast float [[TMP8]] to b32
+; SM90-NEXT: [[TMP16:%.*]] = insertelement <4 x b32> [[TMP14]], b32 [[TMP15]], i32 2
+; SM90-NEXT: [[TMP17:%.*]] = bitcast float [[TMP7]] to b32
+; SM90-NEXT: [[TMP18:%.*]] = insertelement <4 x b32> [[TMP16]], b32 [[TMP17]], i32 3
+; SM90-NEXT: store <4 x b32> [[TMP18]], ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[TMP19:%.*]] = bitcast float [[TMP5]] to b32
+; SM90-NEXT: [[TMP20:%.*]] = insertelement <4 x b32> poison, b32 [[TMP19]], i32 0
+; SM90-NEXT: [[TMP21:%.*]] = bitcast float [[TMP4]] to b32
+; SM90-NEXT: [[TMP22:%.*]] = insertelement <4 x b32> [[TMP20]], b32 [[TMP21]], i32 1
+; SM90-NEXT: [[TMP23:%.*]] = bitcast float [[TMP3]] to b32
+; SM90-NEXT: [[TMP24:%.*]] = insertelement <4 x b32> [[TMP22]], b32 [[TMP23]], i32 2
+; SM90-NEXT: [[TMP25:%.*]] = bitcast float [[TMP2]] to b32
+; SM90-NEXT: [[TMP26:%.*]] = insertelement <4 x b32> [[TMP24]], b32 [[TMP25]], i32 3
+; SM90-NEXT: store <4 x b32> [[TMP26]], ptr addrspace(1) [[PTR4]], align 16
; SM90-NEXT: ret void
;
; SM100-LABEL: define void @float32x8(
; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
; SM100-NEXT: [[PTR0:%.*]] = getelementptr float, ptr addrspace(1) [[PTR]], i64 0
-; SM100-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr addrspace(1) [[PTR0]], align 32
-; SM100-NEXT: [[L01:%.*]] = extractelement <8 x float> [[TMP1]], i32 0
-; SM100-NEXT: [[L12:%.*]] = extractelement <8 x float> [[TMP1]], i32 1
-; SM100-NEXT: [[L23:%.*]] = extractelement <8 x float> [[TMP1]], i32 2
-; SM100-NEXT: [[L34:%.*]] = extractelement <8 x float> [[TMP1]], i32 3
-; SM100-NEXT: [[L45:%.*]] = extractelement <8 x float> [[TMP1]], i32 4
-; SM100-NEXT: [[L56:%.*]] = extractelement <8 x float> [[TMP1]], i32 5
-; SM100-NEXT: [[L67:%.*]] = extractelement <8 x float> [[TMP1]], i32 6
-; SM100-NEXT: [[L78:%.*]] = extractelement <8 x float> [[TMP1]], i32 7
-; SM100-NEXT: [[TMP2:%.*]] = insertelement <8 x float> poison, float [[L78]], i32 0
-; SM100-NEXT: [[TMP3:%.*]] = insertelement <8 x float> [[TMP2]], float [[L67]], i32 1
-; SM100-NEXT: [[TMP4:%.*]] = insertelement <8 x float> [[TMP3]], float [[L56]], i32 2
-; SM100-NEXT: [[TMP5:%.*]] = insertelement <8 x float> [[TMP4]], float [[L45]], i32 3
-; SM100-NEXT: [[TMP6:%.*]] = insertelement <8 x float> [[TMP5]], float [[L34]], i32 4
-; SM100-NEXT: [[TMP7:%.*]] = insertelement <8 x float> [[TMP6]], float [[L23]], i32 5
-; SM100-NEXT: [[TMP8:%.*]] = insertelement <8 x float> [[TMP7]], float [[L12]], i32 6
-; SM100-NEXT: [[TMP9:%.*]] = insertelement <8 x float> [[TMP8]], float [[L01]], i32 7
-; SM100-NEXT: store <8 x float> [[TMP9]], ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[TMP1:%.*]] = load <8 x b32>, ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[L01:%.*]] = extractelement <8 x b32> [[TMP1]], i32 0
+; SM100-NEXT: [[TMP2:%.*]] = bitcast b32 [[L01]] to float
+; SM100-NEXT: [[L12:%.*]] = extractelement <8 x b32> [[TMP1]], i32 1
+; SM100-NEXT: [[TMP3:%.*]] = bitcast b32 [[L12]] to float
+; SM100-NEXT: [[L23:%.*]] = extractelement <8 x b32> [[TMP1]], i32 2
+; SM100-NEXT: [[TMP4:%.*]] = bitcast b32 [[L23]] to float
+; SM100-NEXT: [[L34:%.*]] = extractelement <8 x b32> [[TMP1]], i32 3
+; SM100-NEXT: [[TMP5:%.*]] = bitcast b32 [[L34]] to float
+; SM100-NEXT: [[L45:%.*]] = extractelement <8 x b32> [[TMP1]], i32 4
+; SM100-NEXT: [[TMP6:%.*]] = bitcast b32 [[L45]] to float
+; SM100-NEXT: [[L56:%.*]] = extractelement <8 x b32> [[TMP1]], i32 5
+; SM100-NEXT: [[TMP7:%.*]] = bitcast b32 [[L56]] to float
+; SM100-NEXT: [[L67:%.*]] = extractelement <8 x b32> [[TMP1]], i32 6
+; SM100-NEXT: [[TMP8:%.*]] = bitcast b32 [[L67]] to float
+; SM100-NEXT: [[L78:%.*]] = extractelement <8 x b32> [[TMP1]], i32 7
+; SM100-NEXT: [[TMP9:%.*]] = bitcast b32 [[L78]] to float
+; SM100-NEXT: [[TMP10:%.*]] = bitcast float [[TMP9]] to b32
+; SM100-NEXT: [[TMP11:%.*]] = insertelement <8 x b32> poison, b32 [[TMP10]], i32 0
+; SM100-NEXT: [[TMP12:%.*]] = bitcast float [[TMP8]] to b32
+; SM100-NEXT: [[TMP13:%.*]] = insertelement <8 x b32> [[TMP11]], b32 [[TMP12]], i32 1
+; SM100-NEXT: [[TMP14:%.*]] = bitcast float [[TMP7]] to b32
+; SM100-NEXT: [[TMP15:%.*]] = insertelement <8 x b32> [[TMP13]], b32 [[TMP14]], i32 2
+; SM100-NEXT: [[TMP16:%.*]] = bitcast float [[TMP6]] to b32
+; SM100-NEXT: [[TMP17:%.*]] = insertelement <8 x b32> [[TMP15]], b32 [[TMP16]], i32 3
+; SM100-NEXT: [[TMP18:%.*]] = bitcast float [[TMP5]] to b32
+; SM100-NEXT: [[TMP19:%.*]] = insertelement <8 x b32> [[TMP17]], b32 [[TMP18]], i32 4
+; SM100-NEXT: [[TMP20:%.*]] = bitcast float [[TMP4]] to b32
+; SM100-NEXT: [[TMP21:%.*]] = insertelement <8 x b32> [[TMP19]], b32 [[TMP20]], i32 5
+; SM100-NEXT: [[TMP22:%.*]] = bitcast float [[TMP3]] to b32
+; SM100-NEXT: [[TMP23:%.*]] = insertelement <8 x b32> [[TMP21]], b32 [[TMP22]], i32 6
+; SM100-NEXT: [[TMP24:%.*]] = bitcast float [[TMP2]] to b32
+; SM100-NEXT: [[TMP25:%.*]] = insertelement <8 x b32> [[TMP23]], b32 [[TMP24]], i32 7
+; SM100-NEXT: store <8 x b32> [[TMP25]], ptr addrspace(1) [[PTR0]], align 32
; SM100-NEXT: ret void
;
@@ -622,33 +894,49 @@ define void @float64x4(ptr addrspace(1) %ptr) {
; SM90-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
; SM90-NEXT: [[PTR0:%.*]] = getelementptr double, ptr addrspace(1) [[PTR]], i64 0
; SM90-NEXT: [[PTR2:%.*]] = getelementptr double, ptr addrspace(1) [[PTR]], i64 2
-; SM90-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[L01:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
-; SM90-NEXT: [[L12:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
-; SM90-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr addrspace(1) [[PTR2]], align 16
-; SM90-NEXT: [[L23:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
-; SM90-NEXT: [[L34:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
-; SM90-NEXT: [[TMP3:%.*]] = insertelement <2 x double> poison, double [[L34]], i32 0
-; SM90-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[L23]], i32 1
-; SM90-NEXT: store <2 x double> [[TMP4]], ptr addrspace(1) [[PTR0]], align 32
-; SM90-NEXT: [[TMP5:%.*]] = insertelement <2 x double> poison, double [[L12]], i32 0
-; SM90-NEXT: [[TMP6:%.*]] = insertelement <2 x double> [[TMP5]], double [[L01]], i32 1
-; SM90-NEXT: store <2 x double> [[TMP6]], ptr addrspace(1) [[PTR2]], align 16
+; SM90-NEXT: [[TMP1:%.*]] = load <2 x b64>, ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[L01:%.*]] = extractelement <2 x b64> [[TMP1]], i32 0
+; SM90-NEXT: [[TMP2:%.*]] = bitcast b64 [[L01]] to double
+; SM90-NEXT: [[L12:%.*]] = extractelement <2 x b64> [[TMP1]], i32 1
+; SM90-NEXT: [[TMP3:%.*]] = bitcast b64 [[L12]] to double
+; SM90-NEXT: [[TMP4:%.*]] = load <2 x b64>, ptr addrspace(1) [[PTR2]], align 16
+; SM90-NEXT: [[L23:%.*]] = extractelement <2 x b64> [[TMP4]], i32 0
+; SM90-NEXT: [[TMP5:%.*]] = bitcast b64 [[L23]] to double
+; SM90-NEXT: [[L34:%.*]] = extractelement <2 x b64> [[TMP4]], i32 1
+; SM90-NEXT: [[TMP6:%.*]] = bitcast b64 [[L34]] to double
+; SM90-NEXT: [[TMP7:%.*]] = bitcast double [[TMP6]] to b64
+; SM90-NEXT: [[TMP8:%.*]] = insertelement <2 x b64> poison, b64 [[TMP7]], i32 0
+; SM90-NEXT: [[TMP9:%.*]] = bitcast double [[TMP5]] to b64
+; SM90-NEXT: [[TMP10:%.*]] = insertelement <2 x b64> [[TMP8]], b64 [[TMP9]], i32 1
+; SM90-NEXT: store <2 x b64> [[TMP10]], ptr addrspace(1) [[PTR0]], align 32
+; SM90-NEXT: [[TMP11:%.*]] = bitcast double [[TMP3]] to b64
+; SM90-NEXT: [[TMP12:%.*]] = insertelement <2 x b64> poison, b64 [[TMP11]], i32 0
+; SM90-NEXT: [[TMP13:%.*]] = bitcast double [[TMP2]] to b64
+; SM90-NEXT: [[TMP14:%.*]] = insertelement <2 x b64> [[TMP12]], b64 [[TMP13]], i32 1
+; SM90-NEXT: store <2 x b64> [[TMP14]], ptr addrspace(1) [[PTR2]], align 16
; SM90-NEXT: ret void
;
; SM100-LABEL: define void @float64x4(
; SM100-SAME: ptr addrspace(1) [[PTR:%.*]]) #[[ATTR0]] {
; SM100-NEXT: [[PTR0:%.*]] = getelementptr double, ptr addrspace(1) [[PTR]], i64 0
-; SM100-NEXT: [[TMP1:%.*]] = load <4 x double>, ptr addrspace(1) [[PTR0]], align 32
-; SM100-NEXT: [[L01:%.*]] = extractelement <4 x double> [[TMP1]], i32 0
-; SM100-NEXT: [[L12:%.*]] = extractelement <4 x double> [[TMP1]], i32 1
-; SM100-NEXT: [[L23:%.*]] = extractelement <4 x double> [[TMP1]], i32 2
-; SM100-NEXT: [[L34:%.*]] = extractelement <4 x double> [[TMP1]], i32 3
-; SM100-NEXT: [[TMP2:%.*]] = insertelement <4 x double> poison, double [[L34]], i32 0
-; SM100-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[L23]], i32 1
-; SM100-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[L12]], i32 2
-; SM100-NEXT: [[TMP5:%.*]] = insertelement <4 x double> [[TMP4]], double [[L01]], i32 3
-; SM100-NEXT: store <4 x double> [[TMP5]], ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[TMP1:%.*]] = load <4 x b64>, ptr addrspace(1) [[PTR0]], align 32
+; SM100-NEXT: [[L01:%.*]] = extractelement <4 x b64> [[TMP1]], i32 0
+; SM100-NEXT: [[TMP2:%.*]] = bitcast b64 [[L01]] to double
+; SM100-NEXT: [[L12:%.*]] = extractelement <4 x b64> [[TMP1]], i32 1
+; SM100-NEXT: [[TMP3:%.*]] = bitcast b64 [[L12]] to double
+; SM100-NEXT: [[L23:%.*]] = extractelement <4 x b64> [[TMP1]], i32 2
+; SM100-NEXT: [[TMP4:%.*]] = bitcast b64 [[L23]] to double
+; SM100-NEXT: [[L34:%.*]] = extractelement <4 x b64> [[TMP1]], i32 3
+; SM100-NEXT: [[TMP5:%.*]] = bitcast b64 [[L34]] to double
+; SM100-NEXT: [[TMP6:%.*]] = bitcast double [[TMP5]] to b64
+; SM100-NEXT: [[TMP7:%.*]] = insertelement <4 x b64> poison, b64 [[TMP6]], i32 0
+; SM100-NEXT: [[TMP8:%.*]] = bitcast double [[TMP4]] to b64
+; SM100-NEXT: [[TMP9:%.*]] = insertelement <4 x b64> [[TMP7]], b64 [[TMP8]], i32 1
+; SM100-NEXT: [[TMP10:%.*]] = bitcast double [[TMP3]] to b64
+; SM100-NEXT: [[TMP11:%.*]] = insertelement <4 x b64> [[TMP9]], b64 [[TMP10]], i32 2
+; SM100-NEXT: [[TMP12:%.*]] = bitcast double [[TMP2]] to b64
+; SM100-NEXT: [[TMP13:%.*]] = insertelement <4 x b64> [[TMP11]], b64 [[TMP12]], i32 3
+; SM100-NEXT: store <4 x b64> [[TMP13]], ptr addrspace(1) [[PTR0]], align 32
; SM100-NEXT: ret void
;
@@ -675,27 +963,44 @@ define void @int32x8_non_global(ptr %ptr) {
; CHECK-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i32, ptr [[PTR]], i64 0
; CHECK-NEXT: [[PTR4:%.*]] = getelementptr i32, ptr [[PTR]], i64 4
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[PTR0]], align 32
-; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[L34:%.*]] = extractelement <4 x i32> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i32>, ptr [[PTR4]], align 16
-; CHECK-NEXT: [[L45:%.*]] = extractelement <4 x i32> [[TMP2]], i32 0
-; CHECK-NEXT: [[L56:%.*]] = extractelement <4 x i32> [[TMP2]], i32 1
-; CHECK-NEXT: [[L67:%.*]] = extractelement <4 x i32> [[TMP2]], i32 2
-; CHECK-NEXT: [[L78:%.*]] = extractelement <4 x i32> [[TMP2]], i32 3
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x i32> poison, i32 [[L78]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i32> [[TMP3]], i32 [[L67]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> [[TMP4]], i32 [[L56]], i32 2
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> [[TMP5]], i32 [[L45]], i32 3
-; CHECK-NEXT: store <4 x i32> [[TMP6]], ptr [[PTR0]], align 32
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[L34]], i32 0
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[L23]], i32 1
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[L12]], i32 2
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i32> [[TMP9]], i32 [[L01]], i32 3
-; CHECK-NEXT: store <4 x i32> [[TMP10]], ptr [[PTR4]], align 16
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr [[PTR0]], align 32
+; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[L01]] to i32
+; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[L12]] to i32
+; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[L23]] to i32
+; CHECK-NEXT: [[L34:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[L34]] to i32
+; CHECK-NEXT: [[TMP6:%.*]] = load <4 x b32>, ptr [[PTR4]], align 16
+; CHECK-NEXT: [[L45:%.*]] = extractelement <4 x b32> [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b32 [[L45]] to i32
+; CHECK-NEXT: [[L56:%.*]] = extractelement <4 x b32> [[TMP6]], i32 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b32 [[L56]] to i32
+; CHECK-NEXT: [[L67:%.*]] = extractelement <4 x b32> [[TMP6]], i32 2
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast b32 [[L67]] to i32
+; CHECK-NEXT: [[L78:%.*]] = extractelement <4 x b32> [[TMP6]], i32 3
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast b32 [[L78]] to i32
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32 [[TMP10]] to b32
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x b32> poison, b32 [[TMP11]], i32 0
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast i32 [[TMP9]] to b32
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x b32> [[TMP12]], b32 [[TMP13]], i32 1
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast i32 [[TMP8]] to b32
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x b32> [[TMP14]], b32 [[TMP15]], i32 2
+; CHECK-NEXT: [[TMP17:%.*]] = bitcast i32 [[TMP7]] to b32
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x b32> [[TMP16]], b32 [[TMP17]], i32 3
+; CHECK-NEXT: store <4 x b32> [[TMP18]], ptr [[PTR0]], align 32
+; CHECK-NEXT: [[TMP19:%.*]] = bitcast i32 [[TMP5]] to b32
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x b32> poison, b32 [[TMP19]], i32 0
+; CHECK-NEXT: [[TMP21:%.*]] = bitcast i32 [[TMP4]] to b32
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x b32> [[TMP20]], b32 [[TMP21]], i32 1
+; CHECK-NEXT: [[TMP23:%.*]] = bitcast i32 [[TMP3]] to b32
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x b32> [[TMP22]], b32 [[TMP23]], i32 2
+; CHECK-NEXT: [[TMP25:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x b32> [[TMP24]], b32 [[TMP25]], i32 3
+; CHECK-NEXT: store <4 x b32> [[TMP26]], ptr [[PTR4]], align 16
; CHECK-NEXT: ret void
+;
%ptr0 = getelementptr i32, ptr %ptr, i64 0
%ptr1 = getelementptr i32, ptr %ptr, i64 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/many_loads_stores.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/many_loads_stores.ll
index abe15c00c494b..7faaacfcdef4d 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/many_loads_stores.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/many_loads_stores.ll
@@ -1,10 +1,11 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; This is an end-to-end test that checks that LSV succeeds at vectorizing a
; large program with many loads.
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s > %t
; RUN: grep 'load i8' < %t | count 12
-; RUN: grep 'load <2 x i8>' < %t | count 3
-; RUN: grep 'load <4 x i8>' < %t | count 27
-; RUN: grep 'call <4 x i8> @llvm.masked.load.v4i8.p1.*<4 x i1> <i1 false, i1 true, i1 true, i1 true>' < %t | count 6
+; RUN: grep 'load <2 x b8>' < %t | count 3
+; RUN: grep 'load <4 x b8>' < %t | count 27
+; RUN: grep 'call <4 x b8> @llvm.masked.load.v4b8.p1.*<4 x i1> <i1 false, i1 true, i1 true, i1 true>' < %t | count 6
target datalayout = "e-i64:64-i128:128-v16:16-v32:32-n16:32:64"
target triple = "nvptx64-nvidia-cuda"
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/masked-store.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/masked-store.ll
index a9e9cf674c72e..5653f9a5e918e 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/masked-store.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/masked-store.ll
@@ -7,7 +7,7 @@
define void @singleGap(ptr addrspace(1) %out) {
; CHECK-LABEL: define void @singleGap(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> <i64 1, i64 2, i64 poison, i64 4>, ptr addrspace(1) align 32 [[OUT]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT: call void @llvm.masked.store.v4b64.p1(<4 x b64> <b64 1, b64 2, b64 poison, b64 4>, ptr addrspace(1) align 32 [[OUT]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
store i64 1, ptr addrspace(1) %out, align 32
@@ -22,7 +22,7 @@ define void @singleGap(ptr addrspace(1) %out) {
define void @singleGapDouble(ptr addrspace(1) %out) {
; CHECK-LABEL: define void @singleGapDouble(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: call void @llvm.masked.store.v4f64.p1(<4 x double> <double 1.000000e+00, double 2.000000e+00, double poison, double 4.000000e+00>, ptr addrspace(1) align 32 [[OUT]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
+; CHECK-NEXT: call void @llvm.masked.store.v4b64.p1(<4 x b64> <b64 4607182418800017408, b64 4611686018427387904, b64 poison, b64 4616189618054758400>, ptr addrspace(1) align 32 [[OUT]], <4 x i1> <i1 true, i1 true, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
store double 1.0, ptr addrspace(1) %out, align 32
@@ -37,7 +37,7 @@ define void @singleGapDouble(ptr addrspace(1) %out) {
define void @multipleGaps(ptr addrspace(1) %out) {
; CHECK-LABEL: define void @multipleGaps(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> <i64 1, i64 poison, i64 poison, i64 4>, ptr addrspace(1) align 32 [[OUT]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>)
+; CHECK-NEXT: call void @llvm.masked.store.v4b64.p1(<4 x b64> <b64 1, b64 poison, b64 poison, b64 4>, ptr addrspace(1) align 32 [[OUT]], <4 x i1> <i1 true, i1 false, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
store i64 1, ptr addrspace(1) %out, align 32
@@ -50,7 +50,7 @@ define void @multipleGaps(ptr addrspace(1) %out) {
define void @multipleGaps8xi32(ptr addrspace(1) %out) {
; CHECK-LABEL: define void @multipleGaps8xi32(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p1(<8 x i32> <i32 1, i32 poison, i32 poison, i32 2, i32 4, i32 poison, i32 poison, i32 8>, ptr addrspace(1) align 32 [[OUT]], <8 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true>)
+; CHECK-NEXT: call void @llvm.masked.store.v8b32.p1(<8 x b32> <b32 1, b32 poison, b32 poison, b32 2, b32 4, b32 poison, b32 poison, b32 8>, ptr addrspace(1) align 32 [[OUT]], <8 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true>)
; CHECK-NEXT: ret void
;
store i32 1, ptr addrspace(1) %out, align 32
@@ -69,9 +69,9 @@ define void @singleGapLongerChain(ptr addrspace(1) %out) {
; CHECK-LABEL: define void @singleGapLongerChain(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[GETELEM3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i32 24
-; CHECK-NEXT: store <4 x i64> <i64 1, i64 2, i64 3, i64 4>, ptr addrspace(1) [[OUT]], align 32
+; CHECK-NEXT: store <4 x b64> <b64 1, b64 2, b64 3, b64 4>, ptr addrspace(1) [[OUT]], align 32
; CHECK-NEXT: [[GAPFILLGEP:%.*]] = getelementptr i8, ptr addrspace(1) [[GETELEM3]], i64 8
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> <i64 poison, i64 6, i64 7, i64 8>, ptr addrspace(1) align 32 [[GAPFILLGEP]], <4 x i1> <i1 false, i1 true, i1 true, i1 true>)
+; CHECK-NEXT: call void @llvm.masked.store.v4b64.p1(<4 x b64> <b64 poison, b64 6, b64 7, b64 8>, ptr addrspace(1) align 32 [[GAPFILLGEP]], <4 x i1> <i1 false, i1 true, i1 true, i1 true>)
; CHECK-NEXT: ret void
;
store i64 1, ptr addrspace(1) %out, align 32
@@ -94,7 +94,7 @@ define void @singleGapLongerChain(ptr addrspace(1) %out) {
define void @vectorElements(ptr addrspace(1) %out) {
; CHECK-LABEL: define void @vectorElements(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p1(<8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 poison, i32 poison, i32 7, i32 8>, ptr addrspace(1) align 32 [[OUT]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true>)
+; CHECK-NEXT: call void @llvm.masked.store.v8b32.p1(<8 x b32> <b32 1, b32 2, b32 3, b32 4, b32 poison, b32 poison, b32 7, b32 8>, ptr addrspace(1) align 32 [[OUT]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true>)
; CHECK-NEXT: ret void
;
store <2 x i32> <i32 1, i32 2>, ptr addrspace(1) %out, align 32
@@ -109,7 +109,7 @@ define void @vectorElements(ptr addrspace(1) %out) {
define void @vectorElements64(ptr addrspace(1) %in) {
; CHECK-LABEL: define void @vectorElements64(
; CHECK-SAME: ptr addrspace(1) [[IN:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: store <4 x i64> <i64 1, i64 2, i64 3, i64 4>, ptr addrspace(1) [[IN]], align 32
+; CHECK-NEXT: store <4 x b64> <b64 1, b64 2, b64 3, b64 4>, ptr addrspace(1) [[IN]], align 32
; CHECK-NEXT: [[GETELEM1:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[IN]], i32 48
; CHECK-NEXT: store <2 x i64> <i64 7, i64 8>, ptr addrspace(1) [[GETELEM1]], align 16
; CHECK-NEXT: ret void
@@ -126,7 +126,7 @@ define void @vectorElements64(ptr addrspace(1) %in) {
define void @extendStores(ptr addrspace(1) %out) {
; CHECK-LABEL: define void @extendStores(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: call void @llvm.masked.store.v4i64.p1(<4 x i64> <i64 1, i64 2, i64 3, i64 poison>, ptr addrspace(1) align 32 [[OUT]], <4 x i1> <i1 true, i1 true, i1 true, i1 false>)
+; CHECK-NEXT: call void @llvm.masked.store.v4b64.p1(<4 x b64> <b64 1, b64 2, b64 3, b64 poison>, ptr addrspace(1) align 32 [[OUT]], <4 x i1> <i1 true, i1 true, i1 true, i1 false>)
; CHECK-NEXT: ret void
;
store i64 1, ptr addrspace(1) %out, align 32
@@ -141,7 +141,7 @@ define void @extendStores(ptr addrspace(1) %out) {
define void @extendStores8xi32(ptr addrspace(1) %out) {
; CHECK-LABEL: define void @extendStores8xi32(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p1(<8 x i32> <i32 1, i32 2, i32 3, i32 4, i32 5, i32 poison, i32 poison, i32 poison>, ptr addrspace(1) align 32 [[OUT]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT: call void @llvm.masked.store.v8b32.p1(<8 x b32> <b32 1, b32 2, b32 3, b32 4, b32 5, b32 poison, b32 poison, b32 poison>, ptr addrspace(1) align 32 [[OUT]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>)
; CHECK-NEXT: ret void
;
store i32 1, ptr addrspace(1) %out, align 32
@@ -160,24 +160,34 @@ define void @extendStores8xi32(ptr addrspace(1) %out) {
define void @extendStoresFromLoads8xi32(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; CHECK-LABEL: define void @extendStoresFromLoads8xi32(
; CHECK-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1(ptr addrspace(1) align 32 [[IN]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>, <8 x i32> poison)
-; CHECK-NEXT: [[LOAD05:%.*]] = extractelement <8 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LOAD16:%.*]] = extractelement <8 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[LOAD27:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[LOAD38:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3
-; CHECK-NEXT: [[LOAD49:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4
-; CHECK-NEXT: [[EXTENDLOAD10:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5
-; CHECK-NEXT: [[EXTENDLOAD211:%.*]] = extractelement <8 x i32> [[TMP1]], i32 6
-; CHECK-NEXT: [[EXTENDLOAD412:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[LOAD05]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[LOAD16]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 [[LOAD27]], i32 2
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[LOAD38]], i32 3
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[LOAD49]], i32 4
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 poison, i32 5
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 poison, i32 6
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 poison, i32 7
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p1(<8 x i32> [[TMP9]], ptr addrspace(1) align 32 [[OUT]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x b32> @llvm.masked.load.v8b32.p1(ptr addrspace(1) align 32 [[IN]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>, <8 x b32> poison)
+; CHECK-NEXT: [[LOAD05:%.*]] = extractelement <8 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LOAD05]] to i32
+; CHECK-NEXT: [[LOAD16:%.*]] = extractelement <8 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LOAD16]] to i32
+; CHECK-NEXT: [[LOAD27:%.*]] = extractelement <8 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[LOAD27]] to i32
+; CHECK-NEXT: [[LOAD38:%.*]] = extractelement <8 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LOAD38]] to i32
+; CHECK-NEXT: [[LOAD49:%.*]] = extractelement <8 x b32> [[TMP1]], i32 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[LOAD49]] to i32
+; CHECK-NEXT: [[EXTEND10:%.*]] = extractelement <8 x b32> [[TMP1]], i32 5
+; CHECK-NEXT: [[EXTEND211:%.*]] = extractelement <8 x b32> [[TMP1]], i32 6
+; CHECK-NEXT: [[EXTEND412:%.*]] = extractelement <8 x b32> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x b32> poison, b32 [[TMP7]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast i32 [[TMP3]] to b32
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x b32> [[TMP8]], b32 [[TMP9]], i32 1
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast i32 [[TMP4]] to b32
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <8 x b32> [[TMP10]], b32 [[TMP11]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast i32 [[TMP5]] to b32
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <8 x b32> [[TMP12]], b32 [[TMP13]], i32 3
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast i32 [[TMP6]] to b32
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x b32> [[TMP14]], b32 [[TMP15]], i32 4
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x b32> [[TMP16]], b32 poison, i32 5
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <8 x b32> [[TMP17]], b32 poison, i32 6
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <8 x b32> [[TMP18]], b32 poison, i32 7
+; CHECK-NEXT: call void @llvm.masked.store.v8b32.p1(<8 x b32> [[TMP19]], ptr addrspace(1) align 32 [[OUT]], <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false, i1 false>)
; CHECK-NEXT: ret void
;
%load0 = load i32, ptr addrspace(1) %in, align 32
@@ -206,24 +216,33 @@ define void @extendStoresFromLoads8xi32(ptr addrspace(1) %in, ptr addrspace(1) %
define void @extendAndGapFillStoresFromLoads8xi32(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; CHECK-LABEL: define void @extendAndGapFillStoresFromLoads8xi32(
; CHECK-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <8 x i32> @llvm.masked.load.v8i32.p1(ptr addrspace(1) align 32 [[IN]], <8 x i1> <i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 false>, <8 x i32> poison)
-; CHECK-NEXT: [[LOAD05:%.*]] = extractelement <8 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LOAD16:%.*]] = extractelement <8 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[LOAD27:%.*]] = extractelement <8 x i32> [[TMP1]], i32 2
-; CHECK-NEXT: [[LOAD38:%.*]] = extractelement <8 x i32> [[TMP1]], i32 3
-; CHECK-NEXT: [[LOAD49:%.*]] = extractelement <8 x i32> [[TMP1]], i32 4
-; CHECK-NEXT: [[EXTENDLOAD10:%.*]] = extractelement <8 x i32> [[TMP1]], i32 5
-; CHECK-NEXT: [[EXTENDLOAD211:%.*]] = extractelement <8 x i32> [[TMP1]], i32 6
-; CHECK-NEXT: [[EXTENDLOAD412:%.*]] = extractelement <8 x i32> [[TMP1]], i32 7
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i32> poison, i32 [[LOAD05]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i32> [[TMP2]], i32 [[LOAD16]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i32> [[TMP3]], i32 poison, i32 2
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i32> [[TMP4]], i32 [[LOAD38]], i32 3
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i32> [[TMP5]], i32 [[LOAD49]], i32 4
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x i32> [[TMP6]], i32 poison, i32 5
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i32> [[TMP7]], i32 poison, i32 6
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i32> [[TMP8]], i32 poison, i32 7
-; CHECK-NEXT: call void @llvm.masked.store.v8i32.p1(<8 x i32> [[TMP9]], ptr addrspace(1) align 32 [[OUT]], <8 x i1> <i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 false>)
+; CHECK-NEXT: [[TMP1:%.*]] = call <8 x b32> @llvm.masked.load.v8b32.p1(ptr addrspace(1) align 32 [[IN]], <8 x i1> <i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 false>, <8 x b32> poison)
+; CHECK-NEXT: [[LOAD05:%.*]] = extractelement <8 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LOAD05]] to i32
+; CHECK-NEXT: [[LOAD16:%.*]] = extractelement <8 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LOAD16]] to i32
+; CHECK-NEXT: [[GAPFILL7:%.*]] = extractelement <8 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[GAPFILL7]] to i32
+; CHECK-NEXT: [[LOAD38:%.*]] = extractelement <8 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[LOAD38]] to i32
+; CHECK-NEXT: [[LOAD49:%.*]] = extractelement <8 x b32> [[TMP1]], i32 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[LOAD49]] to i32
+; CHECK-NEXT: [[EXTEND10:%.*]] = extractelement <8 x b32> [[TMP1]], i32 5
+; CHECK-NEXT: [[EXTEND211:%.*]] = extractelement <8 x b32> [[TMP1]], i32 6
+; CHECK-NEXT: [[EXTEND412:%.*]] = extractelement <8 x b32> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32 [[TMP2]] to b32
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x b32> poison, b32 [[TMP7]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast i32 [[TMP3]] to b32
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x b32> [[TMP8]], b32 [[TMP9]], i32 1
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x b32> [[TMP10]], b32 poison, i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast i32 [[TMP5]] to b32
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x b32> [[TMP11]], b32 [[TMP12]], i32 3
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i32 [[TMP6]] to b32
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x b32> [[TMP13]], b32 [[TMP14]], i32 4
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x b32> [[TMP15]], b32 poison, i32 5
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x b32> [[TMP16]], b32 poison, i32 6
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <8 x b32> [[TMP17]], b32 poison, i32 7
+; CHECK-NEXT: call void @llvm.masked.store.v8b32.p1(<8 x b32> [[TMP18]], ptr addrspace(1) align 32 [[OUT]], <8 x i1> <i1 true, i1 true, i1 false, i1 true, i1 true, i1 false, i1 false, i1 false>)
; CHECK-NEXT: ret void
;
%load0 = load i32, ptr addrspace(1) %in, align 32
@@ -251,7 +270,7 @@ define void @extendAndGapFillStoresFromLoads8xi32(ptr addrspace(1) %in, ptr addr
define void @singleGapWrongAddrSpace(ptr addrspace(3) %out) {
; CHECK-LABEL: define void @singleGapWrongAddrSpace(
; CHECK-SAME: ptr addrspace(3) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: store <2 x i64> <i64 1, i64 2>, ptr addrspace(3) [[OUT]], align 32
+; CHECK-NEXT: store <2 x b64> <b64 1, b64 2>, ptr addrspace(3) [[OUT]], align 32
; CHECK-NEXT: [[GETELEM3:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[OUT]], i32 24
; CHECK-NEXT: store i64 4, ptr addrspace(3) [[GETELEM3]], align 8
; CHECK-NEXT: ret void
@@ -268,7 +287,7 @@ define void @singleGapWrongAddrSpace(ptr addrspace(3) %out) {
define void @singleGapMisaligned(ptr addrspace(1) %out) {
; CHECK-LABEL: define void @singleGapMisaligned(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: store <2 x i64> <i64 1, i64 2>, ptr addrspace(1) [[OUT]], align 16
+; CHECK-NEXT: store <2 x b64> <b64 1, b64 2>, ptr addrspace(1) [[OUT]], align 16
; CHECK-NEXT: [[GETELEM3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i32 24
; CHECK-NEXT: store i64 4, ptr addrspace(1) [[GETELEM3]], align 8
; CHECK-NEXT: ret void
@@ -287,7 +306,7 @@ define void @singleGap4xi32(ptr addrspace(1) %out) {
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: store i32 1, ptr addrspace(1) [[OUT]], align 32
; CHECK-NEXT: [[GETELEM2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i32 8
-; CHECK-NEXT: store <2 x i32> <i32 3, i32 4>, ptr addrspace(1) [[GETELEM2]], align 8
+; CHECK-NEXT: store <2 x b32> <b32 3, b32 4>, ptr addrspace(1) [[GETELEM2]], align 8
; CHECK-NEXT: ret void
;
store i32 1, ptr addrspace(1) %out, align 32
@@ -303,13 +322,13 @@ define void @singleGap4xi32(ptr addrspace(1) %out) {
define void @gapInWrongLocation(ptr addrspace(1) %out) {
; CHECK-LABEL: define void @gapInWrongLocation(
; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: store <2 x i32> <i32 1, i32 2>, ptr addrspace(1) [[OUT]], align 32
+; CHECK-NEXT: store <2 x b32> <b32 1, b32 2>, ptr addrspace(1) [[OUT]], align 32
; CHECK-NEXT: [[GETELEM2:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i32 8
; CHECK-NEXT: store i32 3, ptr addrspace(1) [[GETELEM2]], align 8
; CHECK-NEXT: [[GETELEM5:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i32 20
; CHECK-NEXT: store i32 5, ptr addrspace(1) [[GETELEM5]], align 4
; CHECK-NEXT: [[GETELEM6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i32 24
-; CHECK-NEXT: store <2 x i32> <i32 6, i32 7>, ptr addrspace(1) [[GETELEM6]], align 8
+; CHECK-NEXT: store <2 x b32> <b32 6, b32 7>, ptr addrspace(1) [[GETELEM6]], align 8
; CHECK-NEXT: ret void
;
store i32 1, ptr addrspace(1) %out, align 32
@@ -332,39 +351,71 @@ define void @gapInWrongLocation(ptr addrspace(1) %out) {
define void @cantMaski8(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; CHECK-LABEL: define void @cantMaski8(
; CHECK-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <32 x i8> @llvm.masked.load.v32i8.p1(ptr addrspace(1) align 32 [[IN]], <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true>, <32 x i8> poison)
-; CHECK-NEXT: [[LOAD031:%.*]] = extractelement <32 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[GAPFILL32:%.*]] = extractelement <32 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[GAPFILL233:%.*]] = extractelement <32 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[LOAD334:%.*]] = extractelement <32 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[LOAD435:%.*]] = extractelement <32 x i8> [[TMP1]], i32 4
-; CHECK-NEXT: [[GAPFILL436:%.*]] = extractelement <32 x i8> [[TMP1]], i32 5
-; CHECK-NEXT: [[GAPFILL637:%.*]] = extractelement <32 x i8> [[TMP1]], i32 6
-; CHECK-NEXT: [[LOAD738:%.*]] = extractelement <32 x i8> [[TMP1]], i32 7
-; CHECK-NEXT: [[LOAD839:%.*]] = extractelement <32 x i8> [[TMP1]], i32 8
-; CHECK-NEXT: [[GAPFILL840:%.*]] = extractelement <32 x i8> [[TMP1]], i32 9
-; CHECK-NEXT: [[GAPFILL1041:%.*]] = extractelement <32 x i8> [[TMP1]], i32 10
-; CHECK-NEXT: [[LOAD1142:%.*]] = extractelement <32 x i8> [[TMP1]], i32 11
-; CHECK-NEXT: [[LOAD1243:%.*]] = extractelement <32 x i8> [[TMP1]], i32 12
-; CHECK-NEXT: [[GAPFILL1244:%.*]] = extractelement <32 x i8> [[TMP1]], i32 13
-; CHECK-NEXT: [[GAPFILL1445:%.*]] = extractelement <32 x i8> [[TMP1]], i32 14
-; CHECK-NEXT: [[LOAD1546:%.*]] = extractelement <32 x i8> [[TMP1]], i32 15
-; CHECK-NEXT: [[LOAD1647:%.*]] = extractelement <32 x i8> [[TMP1]], i32 16
-; CHECK-NEXT: [[GAPFILL1648:%.*]] = extractelement <32 x i8> [[TMP1]], i32 17
-; CHECK-NEXT: [[GAPFILL1849:%.*]] = extractelement <32 x i8> [[TMP1]], i32 18
-; CHECK-NEXT: [[LOAD1950:%.*]] = extractelement <32 x i8> [[TMP1]], i32 19
-; CHECK-NEXT: [[LOAD2051:%.*]] = extractelement <32 x i8> [[TMP1]], i32 20
-; CHECK-NEXT: [[GAPFILL2052:%.*]] = extractelement <32 x i8> [[TMP1]], i32 21
-; CHECK-NEXT: [[GAPFILL2253:%.*]] = extractelement <32 x i8> [[TMP1]], i32 22
-; CHECK-NEXT: [[LOAD2354:%.*]] = extractelement <32 x i8> [[TMP1]], i32 23
-; CHECK-NEXT: [[LOAD2455:%.*]] = extractelement <32 x i8> [[TMP1]], i32 24
-; CHECK-NEXT: [[GAPFILL2456:%.*]] = extractelement <32 x i8> [[TMP1]], i32 25
-; CHECK-NEXT: [[GAPFILL2657:%.*]] = extractelement <32 x i8> [[TMP1]], i32 26
-; CHECK-NEXT: [[LOAD2758:%.*]] = extractelement <32 x i8> [[TMP1]], i32 27
-; CHECK-NEXT: [[LOAD2859:%.*]] = extractelement <32 x i8> [[TMP1]], i32 28
-; CHECK-NEXT: [[GAPFILL2860:%.*]] = extractelement <32 x i8> [[TMP1]], i32 29
-; CHECK-NEXT: [[GAPFILL3061:%.*]] = extractelement <32 x i8> [[TMP1]], i32 30
-; CHECK-NEXT: [[LOAD3162:%.*]] = extractelement <32 x i8> [[TMP1]], i32 31
+; CHECK-NEXT: [[TMP1:%.*]] = call <32 x b8> @llvm.masked.load.v32b8.p1(ptr addrspace(1) align 32 [[IN]], <32 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true>, <32 x b8> poison)
+; CHECK-NEXT: [[LOAD31:%.*]] = extractelement <32 x b8> [[TMP1]], i32 0
+; CHECK-NEXT: [[LOAD031:%.*]] = bitcast b8 [[LOAD31]] to i8
+; CHECK-NEXT: [[GAPFILL32:%.*]] = extractelement <32 x b8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[GAPFILL32]] to i8
+; CHECK-NEXT: [[GAPFILL233:%.*]] = extractelement <32 x b8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b8 [[GAPFILL233]] to i8
+; CHECK-NEXT: [[LOAD335:%.*]] = extractelement <32 x b8> [[TMP1]], i32 3
+; CHECK-NEXT: [[LOAD334:%.*]] = bitcast b8 [[LOAD335]] to i8
+; CHECK-NEXT: [[LOAD436:%.*]] = extractelement <32 x b8> [[TMP1]], i32 4
+; CHECK-NEXT: [[LOAD435:%.*]] = bitcast b8 [[LOAD436]] to i8
+; CHECK-NEXT: [[GAPFILL436:%.*]] = extractelement <32 x b8> [[TMP1]], i32 5
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b8 [[GAPFILL436]] to i8
+; CHECK-NEXT: [[GAPFILL637:%.*]] = extractelement <32 x b8> [[TMP1]], i32 6
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b8 [[GAPFILL637]] to i8
+; CHECK-NEXT: [[LOAD739:%.*]] = extractelement <32 x b8> [[TMP1]], i32 7
+; CHECK-NEXT: [[LOAD738:%.*]] = bitcast b8 [[LOAD739]] to i8
+; CHECK-NEXT: [[LOAD840:%.*]] = extractelement <32 x b8> [[TMP1]], i32 8
+; CHECK-NEXT: [[LOAD839:%.*]] = bitcast b8 [[LOAD840]] to i8
+; CHECK-NEXT: [[GAPFILL840:%.*]] = extractelement <32 x b8> [[TMP1]], i32 9
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast b8 [[GAPFILL840]] to i8
+; CHECK-NEXT: [[GAPFILL1041:%.*]] = extractelement <32 x b8> [[TMP1]], i32 10
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast b8 [[GAPFILL1041]] to i8
+; CHECK-NEXT: [[LOAD1143:%.*]] = extractelement <32 x b8> [[TMP1]], i32 11
+; CHECK-NEXT: [[LOAD1142:%.*]] = bitcast b8 [[LOAD1143]] to i8
+; CHECK-NEXT: [[LOAD1244:%.*]] = extractelement <32 x b8> [[TMP1]], i32 12
+; CHECK-NEXT: [[LOAD1243:%.*]] = bitcast b8 [[LOAD1244]] to i8
+; CHECK-NEXT: [[GAPFILL1244:%.*]] = extractelement <32 x b8> [[TMP1]], i32 13
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast b8 [[GAPFILL1244]] to i8
+; CHECK-NEXT: [[GAPFILL1445:%.*]] = extractelement <32 x b8> [[TMP1]], i32 14
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast b8 [[GAPFILL1445]] to i8
+; CHECK-NEXT: [[LOAD1547:%.*]] = extractelement <32 x b8> [[TMP1]], i32 15
+; CHECK-NEXT: [[LOAD1546:%.*]] = bitcast b8 [[LOAD1547]] to i8
+; CHECK-NEXT: [[LOAD1648:%.*]] = extractelement <32 x b8> [[TMP1]], i32 16
+; CHECK-NEXT: [[LOAD1647:%.*]] = bitcast b8 [[LOAD1648]] to i8
+; CHECK-NEXT: [[GAPFILL1648:%.*]] = extractelement <32 x b8> [[TMP1]], i32 17
+; CHECK-NEXT: [[TMP19:%.*]] = bitcast b8 [[GAPFILL1648]] to i8
+; CHECK-NEXT: [[GAPFILL1849:%.*]] = extractelement <32 x b8> [[TMP1]], i32 18
+; CHECK-NEXT: [[TMP20:%.*]] = bitcast b8 [[GAPFILL1849]] to i8
+; CHECK-NEXT: [[LOAD1951:%.*]] = extractelement <32 x b8> [[TMP1]], i32 19
+; CHECK-NEXT: [[LOAD1950:%.*]] = bitcast b8 [[LOAD1951]] to i8
+; CHECK-NEXT: [[LOAD2052:%.*]] = extractelement <32 x b8> [[TMP1]], i32 20
+; CHECK-NEXT: [[LOAD2051:%.*]] = bitcast b8 [[LOAD2052]] to i8
+; CHECK-NEXT: [[GAPFILL2052:%.*]] = extractelement <32 x b8> [[TMP1]], i32 21
+; CHECK-NEXT: [[TMP23:%.*]] = bitcast b8 [[GAPFILL2052]] to i8
+; CHECK-NEXT: [[GAPFILL2253:%.*]] = extractelement <32 x b8> [[TMP1]], i32 22
+; CHECK-NEXT: [[TMP24:%.*]] = bitcast b8 [[GAPFILL2253]] to i8
+; CHECK-NEXT: [[LOAD2355:%.*]] = extractelement <32 x b8> [[TMP1]], i32 23
+; CHECK-NEXT: [[LOAD2354:%.*]] = bitcast b8 [[LOAD2355]] to i8
+; CHECK-NEXT: [[LOAD2456:%.*]] = extractelement <32 x b8> [[TMP1]], i32 24
+; CHECK-NEXT: [[LOAD2455:%.*]] = bitcast b8 [[LOAD2456]] to i8
+; CHECK-NEXT: [[GAPFILL2456:%.*]] = extractelement <32 x b8> [[TMP1]], i32 25
+; CHECK-NEXT: [[TMP27:%.*]] = bitcast b8 [[GAPFILL2456]] to i8
+; CHECK-NEXT: [[GAPFILL2657:%.*]] = extractelement <32 x b8> [[TMP1]], i32 26
+; CHECK-NEXT: [[TMP28:%.*]] = bitcast b8 [[GAPFILL2657]] to i8
+; CHECK-NEXT: [[LOAD2759:%.*]] = extractelement <32 x b8> [[TMP1]], i32 27
+; CHECK-NEXT: [[LOAD2758:%.*]] = bitcast b8 [[LOAD2759]] to i8
+; CHECK-NEXT: [[LOAD2860:%.*]] = extractelement <32 x b8> [[TMP1]], i32 28
+; CHECK-NEXT: [[LOAD2859:%.*]] = bitcast b8 [[LOAD2860]] to i8
+; CHECK-NEXT: [[GAPFILL2860:%.*]] = extractelement <32 x b8> [[TMP1]], i32 29
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast b8 [[GAPFILL2860]] to i8
+; CHECK-NEXT: [[GAPFILL3061:%.*]] = extractelement <32 x b8> [[TMP1]], i32 30
+; CHECK-NEXT: [[TMP32:%.*]] = bitcast b8 [[GAPFILL3061]] to i8
+; CHECK-NEXT: [[LOAD3163:%.*]] = extractelement <32 x b8> [[TMP1]], i32 31
+; CHECK-NEXT: [[LOAD3162:%.*]] = bitcast b8 [[LOAD3163]] to i8
; CHECK-NEXT: store i8 [[LOAD031]], ptr addrspace(1) [[OUT]], align 32
; CHECK-NEXT: [[OUTELEM3:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i32 3
; CHECK-NEXT: store i8 [[LOAD334]], ptr addrspace(1) [[OUTELEM3]], align 1
@@ -471,23 +522,39 @@ define void @cantMaski8(ptr addrspace(1) %in, ptr addrspace(1) %out) {
define void @cantMaski16(ptr addrspace(1) %in, ptr addrspace(1) %out) {
; CHECK-LABEL: define void @cantMaski16(
; CHECK-SAME: ptr addrspace(1) [[IN:%.*]], ptr addrspace(1) [[OUT:%.*]]) #[[ATTR0]] {
-; CHECK-NEXT: [[TMP1:%.*]] = call <16 x i16> @llvm.masked.load.v16i16.p1(ptr addrspace(1) align 32 [[IN]], <16 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true>, <16 x i16> poison)
-; CHECK-NEXT: [[LOAD015:%.*]] = extractelement <16 x i16> [[TMP1]], i32 0
-; CHECK-NEXT: [[GAPFILL16:%.*]] = extractelement <16 x i16> [[TMP1]], i32 1
-; CHECK-NEXT: [[GAPFILL217:%.*]] = extractelement <16 x i16> [[TMP1]], i32 2
-; CHECK-NEXT: [[LOAD318:%.*]] = extractelement <16 x i16> [[TMP1]], i32 3
-; CHECK-NEXT: [[LOAD419:%.*]] = extractelement <16 x i16> [[TMP1]], i32 4
-; CHECK-NEXT: [[GAPFILL420:%.*]] = extractelement <16 x i16> [[TMP1]], i32 5
-; CHECK-NEXT: [[GAPFILL621:%.*]] = extractelement <16 x i16> [[TMP1]], i32 6
-; CHECK-NEXT: [[LOAD722:%.*]] = extractelement <16 x i16> [[TMP1]], i32 7
-; CHECK-NEXT: [[LOAD823:%.*]] = extractelement <16 x i16> [[TMP1]], i32 8
-; CHECK-NEXT: [[GAPFILL824:%.*]] = extractelement <16 x i16> [[TMP1]], i32 9
-; CHECK-NEXT: [[GAPFILL1025:%.*]] = extractelement <16 x i16> [[TMP1]], i32 10
-; CHECK-NEXT: [[LOAD1126:%.*]] = extractelement <16 x i16> [[TMP1]], i32 11
-; CHECK-NEXT: [[LOAD1227:%.*]] = extractelement <16 x i16> [[TMP1]], i32 12
-; CHECK-NEXT: [[GAPFILL1228:%.*]] = extractelement <16 x i16> [[TMP1]], i32 13
-; CHECK-NEXT: [[GAPFILL1429:%.*]] = extractelement <16 x i16> [[TMP1]], i32 14
-; CHECK-NEXT: [[LOAD1530:%.*]] = extractelement <16 x i16> [[TMP1]], i32 15
+; CHECK-NEXT: [[TMP1:%.*]] = call <16 x b16> @llvm.masked.load.v16b16.p1(ptr addrspace(1) align 32 [[IN]], <16 x i1> <i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true, i1 true, i1 false, i1 false, i1 true>, <16 x b16> poison)
+; CHECK-NEXT: [[LOAD15:%.*]] = extractelement <16 x b16> [[TMP1]], i32 0
+; CHECK-NEXT: [[LOAD015:%.*]] = bitcast b16 [[LOAD15]] to i16
+; CHECK-NEXT: [[GAPFILL16:%.*]] = extractelement <16 x b16> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b16 [[GAPFILL16]] to i16
+; CHECK-NEXT: [[GAPFILL217:%.*]] = extractelement <16 x b16> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b16 [[GAPFILL217]] to i16
+; CHECK-NEXT: [[LOAD319:%.*]] = extractelement <16 x b16> [[TMP1]], i32 3
+; CHECK-NEXT: [[LOAD318:%.*]] = bitcast b16 [[LOAD319]] to i16
+; CHECK-NEXT: [[LOAD420:%.*]] = extractelement <16 x b16> [[TMP1]], i32 4
+; CHECK-NEXT: [[LOAD419:%.*]] = bitcast b16 [[LOAD420]] to i16
+; CHECK-NEXT: [[GAPFILL420:%.*]] = extractelement <16 x b16> [[TMP1]], i32 5
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b16 [[GAPFILL420]] to i16
+; CHECK-NEXT: [[GAPFILL621:%.*]] = extractelement <16 x b16> [[TMP1]], i32 6
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b16 [[GAPFILL621]] to i16
+; CHECK-NEXT: [[LOAD723:%.*]] = extractelement <16 x b16> [[TMP1]], i32 7
+; CHECK-NEXT: [[LOAD722:%.*]] = bitcast b16 [[LOAD723]] to i16
+; CHECK-NEXT: [[LOAD824:%.*]] = extractelement <16 x b16> [[TMP1]], i32 8
+; CHECK-NEXT: [[LOAD823:%.*]] = bitcast b16 [[LOAD824]] to i16
+; CHECK-NEXT: [[GAPFILL824:%.*]] = extractelement <16 x b16> [[TMP1]], i32 9
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast b16 [[GAPFILL824]] to i16
+; CHECK-NEXT: [[GAPFILL1025:%.*]] = extractelement <16 x b16> [[TMP1]], i32 10
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast b16 [[GAPFILL1025]] to i16
+; CHECK-NEXT: [[LOAD1127:%.*]] = extractelement <16 x b16> [[TMP1]], i32 11
+; CHECK-NEXT: [[LOAD1126:%.*]] = bitcast b16 [[LOAD1127]] to i16
+; CHECK-NEXT: [[LOAD1228:%.*]] = extractelement <16 x b16> [[TMP1]], i32 12
+; CHECK-NEXT: [[LOAD1227:%.*]] = bitcast b16 [[LOAD1228]] to i16
+; CHECK-NEXT: [[GAPFILL1228:%.*]] = extractelement <16 x b16> [[TMP1]], i32 13
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast b16 [[GAPFILL1228]] to i16
+; CHECK-NEXT: [[GAPFILL1429:%.*]] = extractelement <16 x b16> [[TMP1]], i32 14
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast b16 [[GAPFILL1429]] to i16
+; CHECK-NEXT: [[LOAD1531:%.*]] = extractelement <16 x b16> [[TMP1]], i32 15
+; CHECK-NEXT: [[LOAD1530:%.*]] = bitcast b16 [[LOAD1531]] to i16
; CHECK-NEXT: store i16 [[LOAD015]], ptr addrspace(1) [[OUT]], align 32
; CHECK-NEXT: [[OUTELEM6:%.*]] = getelementptr inbounds i8, ptr addrspace(1) [[OUT]], i32 6
; CHECK-NEXT: store i16 [[LOAD318]], ptr addrspace(1) [[OUTELEM6]], align 2
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
index 782dba25f063d..19e633e7d633c 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/merge-across-side-effects.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; Check that the load/store vectorizer is willing to move loads/stores across
@@ -15,11 +16,15 @@ declare void @fn_writeonly() #3
declare void @fn_readonly() #4
declare void @fn_readnone() #5
-; CHECK-LABEL: @load_fn
-; CHECK: load
-; CHECK: call void @fn()
-; CHECK: load
define void @load_fn(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn()
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -28,11 +33,15 @@ define void @load_fn(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind
-; CHECK: load
-; CHECK: call void @fn_nounwind()
-; CHECK: load
define void @load_fn_nounwind(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_nounwind() #[[ATTR0]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -41,11 +50,15 @@ define void @load_fn_nounwind(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind_writeonly
-; CHECK: load
-; CHECK: call void @fn_nounwind_writeonly()
-; CHECK: load
define void @load_fn_nounwind_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_nounwind_writeonly() #[[ATTR1:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -54,10 +67,17 @@ define void @load_fn_nounwind_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_nounwind_readonly
-; CHECK-DAG: load <2 x i32>
-; CHECK-DAG: call void @fn_nounwind_readonly()
define void @load_fn_nounwind_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_nounwind_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[V01]] to i32
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[V12]] to i32
+; CHECK-NEXT: call void @fn_nounwind_readonly() #[[ATTR2:[0-9]+]]
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -66,11 +86,15 @@ define void @load_fn_nounwind_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_readonly
-; CHECK: load
-; CHECK: call void @fn_readonly
-; CHECK: load
define void @load_fn_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_readonly() #[[ATTR4:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -79,11 +103,15 @@ define void @load_fn_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_writeonly
-; CHECK: load
-; CHECK: call void @fn_writeonly()
-; CHECK: load
define void @load_fn_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: [[V0:%.*]] = load i32, ptr [[P]], align 8
+; CHECK-NEXT: call void @fn_writeonly() #[[ATTR3:[0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = load i32, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -92,10 +120,17 @@ define void @load_fn_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @load_fn_readnone
-; CHECK-DAG: load <2 x i32>
-; CHECK-DAG: call void @fn_readnone()
define void @load_fn_readnone(ptr %p) #0 {
+; CHECK-LABEL: define void @load_fn_readnone(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[P]], align 8
+; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[V01]] to i32
+; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[V12]] to i32
+; CHECK-NEXT: call void @fn_readnone() #[[ATTR5:[0-9]+]]
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
%v0 = load i32, ptr %p, align 8
@@ -108,11 +143,15 @@ define void @load_fn_readnone(ptr %p) #0 {
; Same tests, but now for stores instead of loads.
; ------------------------------------------------
-; CHECK-LABEL: @store_fn
-; CHECK: store
-; CHECK: call void @fn()
-; CHECK: store
define void @store_fn(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn()
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -121,11 +160,15 @@ define void @store_fn(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind
-; CHECK: store
-; CHECK: call void @fn_nounwind()
-; CHECK: store
define void @store_fn_nounwind(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind() #[[ATTR0]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -134,11 +177,15 @@ define void @store_fn_nounwind(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind_writeonly
-; CHECK: store
-; CHECK: call void @fn_nounwind_writeonly()
-; CHECK: store
define void @store_fn_nounwind_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind_writeonly() #[[ATTR1]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -147,11 +194,15 @@ define void @store_fn_nounwind_writeonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_nounwind_readonly
-; CHECK: store
-; CHECK: call void @fn_nounwind_readonly()
-; CHECK: store
define void @store_fn_nounwind_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_nounwind_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_nounwind_readonly() #[[ATTR2]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -160,11 +211,15 @@ define void @store_fn_nounwind_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_readonly
-; CHECK: store
-; CHECK: call void @fn_readonly
-; CHECK: store
define void @store_fn_readonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_readonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_readonly() #[[ATTR4]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -173,11 +228,15 @@ define void @store_fn_readonly(ptr %p) #0 {
ret void
}
-; CHECK-LABEL: @store_fn_writeonly
-; CHECK: store
-; CHECK: call void @fn_writeonly()
-; CHECK: store
define void @store_fn_writeonly(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_writeonly(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[P_1:%.*]] = getelementptr i32, ptr [[P]], i32 1
+; CHECK-NEXT: store i32 0, ptr [[P]], align 4
+; CHECK-NEXT: call void @fn_writeonly() #[[ATTR3]]
+; CHECK-NEXT: store i32 0, ptr [[P_1]], align 4
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p
@@ -187,10 +246,13 @@ define void @store_fn_writeonly(ptr %p) #0 {
}
; This is the only store idiom we can vectorize.
-; CHECK-LABEL: @store_fn_readnone
-; CHECK-DAG: store <2 x i32>
-; CHECK-DAG: call void @fn_readnone()
define void @store_fn_readnone(ptr %p) #0 {
+; CHECK-LABEL: define void @store_fn_readnone(
+; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: call void @fn_readnone() #[[ATTR5]]
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr [[P]], align 8
+; CHECK-NEXT: ret void
+;
%p.1 = getelementptr i32, ptr %p, i32 1
store i32 0, ptr %p, align 8
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/non-instr-bitcast.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/non-instr-bitcast.ll
index de8a9904cb582..c6a3d85e5b1a0 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/non-instr-bitcast.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/non-instr-bitcast.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; Load from a constant. This can be vectorized, but shouldn't crash us.
@@ -5,7 +6,18 @@
@global = internal addrspace(1) constant [4 x float] [float 0xBF71111120000000, float 0x3F70410420000000, float 0xBF81111120000000, float 0x3FB5555560000000], align 4
define void @foo() {
- ; CHECK: load <4 x float>
+; CHECK-LABEL: define void @foo() {
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr addrspace(1) @global, align 16
+; CHECK-NEXT: [[A1:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[A1]] to float
+; CHECK-NEXT: [[B2:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[B2]] to float
+; CHECK-NEXT: [[C3:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[C3]] to float
+; CHECK-NEXT: [[D4:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[D4]] to float
+; CHECK-NEXT: ret void
+;
%a = load float, ptr addrspace(1) @global, align 16
%b = load float, ptr addrspace(1) getelementptr inbounds ([4 x float], ptr addrspace(1) @global, i64 0, i64 1), align 4
%c = load float, ptr addrspace(1) getelementptr inbounds ([4 x float], ptr addrspace(1) @global, i64 0, i64 2), align 4
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/overlapping_chains.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/overlapping_chains.ll
index e38abec4a6862..11d8e1da75a40 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/overlapping_chains.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/overlapping_chains.ll
@@ -1,10 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
-; CHECK-LABEL: @overlapping_stores
-; CHECK: store i16
-; CHECK: store i16
-; CHECK: store i16
define void @overlapping_stores(ptr nocapture align 2 %ptr) {
+; CHECK-LABEL: define void @overlapping_stores(
+; CHECK-SAME: ptr align 2 captures(none) [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i16, ptr [[PTR]], i64 0
+; CHECK-NEXT: [[PTR1:%.*]] = getelementptr i8, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[PTR2:%.*]] = getelementptr i16, ptr [[PTR]], i64 1
+; CHECK-NEXT: store i16 0, ptr [[PTR0]], align 2
+; CHECK-NEXT: store i16 0, ptr [[PTR1]], align 1
+; CHECK-NEXT: store i16 0, ptr [[PTR2]], align 2
+; CHECK-NEXT: ret void
+;
%ptr0 = getelementptr i16, ptr %ptr, i64 0
%ptr1 = getelementptr i8, ptr %ptr, i64 1
%ptr2 = getelementptr i16, ptr %ptr, i64 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/propagate-invariance-metadata.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/propagate-invariance-metadata.ll
index f2d4ea20230d7..f46189dc4433b 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/propagate-invariance-metadata.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/propagate-invariance-metadata.ll
@@ -8,9 +8,11 @@ define i32 @foo(ptr %ptr) {
; CHECK-LABEL: @foo(
; CHECK-NEXT: [[PTR1:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i32 1
; CHECK-NEXT: [[P1:%.*]] = addrspacecast ptr [[PTR1]] to ptr addrspace(1)
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[PTR]], align 8, !invariant.load !0
-; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
-; CHECK-NEXT: [[V12:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[PTR]], align 8, !invariant.load [[META0:![0-9]+]]
+; CHECK-NEXT: [[V1:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[V01:%.*]] = bitcast b32 [[V1]] to i32
+; CHECK-NEXT: [[V13:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[V12:%.*]] = bitcast b32 [[V13]] to i32
; CHECK-NEXT: [[SUM:%.*]] = add i32 [[V01]], [[V12]]
; CHECK-NEXT: ret i32 [[SUM]]
;
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i1.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i1.ll
index 6d2a462e25321..cfd12b2f2704f 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i1.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i1.ll
@@ -1,6 +1,17 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
define void @i1x8(ptr nocapture align 4 %ptr) {
+; CHECK-LABEL: define void @i1x8(
+; CHECK-SAME: ptr align 4 captures(none) [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <32 x i1>, ptr [[PTR0]], align 4
+; CHECK-NEXT: [[L01:%.*]] = shufflevector <32 x i1> [[TMP1]], <32 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[L12:%.*]] = shufflevector <32 x i1> [[TMP1]], <32 x i1> poison, <8 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; CHECK-NEXT: [[L23:%.*]] = shufflevector <32 x i1> [[TMP1]], <32 x i1> poison, <8 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT: [[L34:%.*]] = shufflevector <32 x i1> [[TMP1]], <32 x i1> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: ret void
+;
%ptr0 = getelementptr i8, ptr %ptr, i64 0
%ptr1 = getelementptr i8, ptr %ptr, i64 1
%ptr2 = getelementptr i8, ptr %ptr, i64 2
@@ -13,11 +24,18 @@ define void @i1x8(ptr nocapture align 4 %ptr) {
ret void
-; CHECK-LABEL: @i1x8
-; CHECK-DAG: load <32 x i1>
}
define void @i1x8x16x8(ptr nocapture align 4 %ptr) {
+; CHECK-LABEL: define void @i1x8x16x8(
+; CHECK-SAME: ptr align 4 captures(none) [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <32 x i1>, ptr [[PTR0]], align 4
+; CHECK-NEXT: [[L01:%.*]] = shufflevector <32 x i1> [[TMP1]], <32 x i1> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; CHECK-NEXT: [[L22:%.*]] = shufflevector <32 x i1> [[TMP1]], <32 x i1> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23>
+; CHECK-NEXT: [[L33:%.*]] = shufflevector <32 x i1> [[TMP1]], <32 x i1> poison, <8 x i32> <i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; CHECK-NEXT: ret void
+;
%ptr0 = getelementptr i8, ptr %ptr, i64 0
%ptr1 = getelementptr i8, ptr %ptr, i64 1
%ptr2 = getelementptr i8, ptr %ptr, i64 3
@@ -28,6 +46,4 @@ define void @i1x8x16x8(ptr nocapture align 4 %ptr) {
ret void
-; CHECK-LABEL: @i1x8x16x8
-; CHECK-DAG: load <32 x i1>
}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i16.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i16.ll
index 8b7537680f892..6d2348e7a3fc4 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i16.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i16.ll
@@ -1,9 +1,22 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
-; CHECK-LABEL: @int16x2
-; CHECK: load <2 x i16>
-; CHECK: store <2 x i16>
define void @int16x2(ptr nocapture align 4 %ptr) {
+; CHECK-LABEL: define void @int16x2(
+; CHECK-SAME: ptr align 4 captures(none) [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i16, ptr [[PTR]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b16>, ptr [[PTR0]], align 4
+; CHECK-NEXT: [[L01:%.*]] = extractelement <2 x b16> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b16 [[L01]] to i16
+; CHECK-NEXT: [[L12:%.*]] = extractelement <2 x b16> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b16 [[L12]] to i16
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i16 [[TMP3]] to b16
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x b16> poison, b16 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i16 [[TMP2]] to b16
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x b16> [[TMP5]], b16 [[TMP6]], i32 1
+; CHECK-NEXT: store <2 x b16> [[TMP7]], ptr [[PTR0]], align 4
+; CHECK-NEXT: ret void
+;
%ptr0 = getelementptr i16, ptr %ptr, i64 0
%ptr1 = getelementptr i16, ptr %ptr, i64 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i24.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i24.ll
index c90301d868967..21de9988cf3ad 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i24.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i24.ll
@@ -1,13 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; We don't need to vectorize this. Just make sure it doesn't crash.
-; CHECK-LABEL: @int24x2
-; CHECK: load i24
-; CHECK: load i24
-; CHECK: store i24
-; CHECK: store i24
define void @int24x2(ptr nocapture align 4 %ptr) {
+; CHECK-LABEL: define void @int24x2(
+; CHECK-SAME: ptr align 4 captures(none) [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i24, ptr [[PTR]], i64 0
+; CHECK-NEXT: [[PTR1:%.*]] = getelementptr i24, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[L0:%.*]] = load i24, ptr [[PTR0]], align 4
+; CHECK-NEXT: [[L1:%.*]] = load i24, ptr [[PTR1]], align 1
+; CHECK-NEXT: store i24 [[L1]], ptr [[PTR0]], align 4
+; CHECK-NEXT: store i24 [[L0]], ptr [[PTR1]], align 1
+; CHECK-NEXT: ret void
+;
%ptr0 = getelementptr i24, ptr %ptr, i64 0
%ptr1 = getelementptr i24, ptr %ptr, i64 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i8.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i8.ll
index 47c9d9a88d958..b2192ce5d1b49 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i8.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_i8.ll
@@ -8,13 +8,17 @@ define void @int8x3a2(ptr nocapture align 2 %ptr) {
; CHECK-SAME: ptr align 2 captures(none) [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
; CHECK-NEXT: [[PTR2:%.*]] = getelementptr i8, ptr [[PTR]], i64 2
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr [[PTR0]], align 2
-; CHECK-NEXT: [[L01:%.*]] = extractelement <2 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <2 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b8>, ptr [[PTR0]], align 2
+; CHECK-NEXT: [[L1:%.*]] = extractelement <2 x b8> [[TMP1]], i32 0
+; CHECK-NEXT: [[L01:%.*]] = bitcast b8 [[L1]] to i8
+; CHECK-NEXT: [[L12:%.*]] = extractelement <2 x b8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[L12]] to i8
; CHECK-NEXT: [[L2:%.*]] = load i8, ptr [[PTR2]], align 2
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i8> poison, i8 [[L2]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i8> [[TMP2]], i8 [[L12]], i32 1
-; CHECK-NEXT: store <2 x i8> [[TMP3]], ptr [[PTR0]], align 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast i8 [[L2]] to b8
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x b8> poison, b8 [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast i8 [[TMP3]] to b8
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <2 x b8> [[TMP5]], b8 [[TMP6]], i32 1
+; CHECK-NEXT: store <2 x b8> [[TMP7]], ptr [[PTR0]], align 2
; CHECK-NEXT: store i8 [[L01]], ptr [[PTR2]], align 2
; CHECK-NEXT: ret void
;
@@ -39,14 +43,19 @@ define void @int8x3a4(ptr nocapture align 4 %ptr) {
; CHECK-SAME: ptr align 4 captures(none) [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
; CHECK-NEXT: [[PTR2:%.*]] = getelementptr i8, ptr [[PTR]], i64 2
-; CHECK-NEXT: [[TMP1:%.*]] = call <4 x i8> @llvm.masked.load.v4i8.p0(ptr align 4 [[PTR0]], <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x i8> poison)
-; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[EXTEND4:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x i8> poison, i8 [[L23]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x i8> [[TMP2]], i8 [[L12]], i32 1
-; CHECK-NEXT: store <2 x i8> [[TMP3]], ptr [[PTR0]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = call <4 x b8> @llvm.masked.load.v4b8.p0(ptr align 4 [[PTR0]], <4 x i1> <i1 true, i1 true, i1 true, i1 false>, <4 x b8> poison)
+; CHECK-NEXT: [[L1:%.*]] = extractelement <4 x b8> [[TMP1]], i32 0
+; CHECK-NEXT: [[L01:%.*]] = bitcast b8 [[L1]] to i8
+; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x b8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[L12]] to i8
+; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x b8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b8 [[L23]] to i8
+; CHECK-NEXT: [[EXTEND4:%.*]] = extractelement <4 x b8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[TMP4]] to b8
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x b8> poison, b8 [[TMP5]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i8 [[TMP3]] to b8
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x b8> [[TMP6]], b8 [[TMP7]], i32 1
+; CHECK-NEXT: store <2 x b8> [[TMP8]], ptr [[PTR0]], align 4
; CHECK-NEXT: store i8 [[L01]], ptr [[PTR2]], align 2
; CHECK-NEXT: ret void
;
@@ -72,36 +81,60 @@ define void @int8x12a4(ptr nocapture align 4 %ptr) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
; CHECK-NEXT: [[PTR4:%.*]] = getelementptr i8, ptr [[PTR]], i64 4
; CHECK-NEXT: [[PTR8:%.*]] = getelementptr i8, ptr [[PTR]], i64 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[PTR0]], align 4
-; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[L34:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[PTR4]], align 4
-; CHECK-NEXT: [[L45:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
-; CHECK-NEXT: [[L56:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
-; CHECK-NEXT: [[L67:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
-; CHECK-NEXT: [[L78:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
-; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[PTR8]], align 4
-; CHECK-NEXT: [[L89:%.*]] = extractelement <4 x i8> [[TMP3]], i32 0
-; CHECK-NEXT: [[L910:%.*]] = extractelement <4 x i8> [[TMP3]], i32 1
-; CHECK-NEXT: [[LA11:%.*]] = extractelement <4 x i8> [[TMP3]], i32 2
-; CHECK-NEXT: [[LB12:%.*]] = extractelement <4 x i8> [[TMP3]], i32 3
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x i8> poison, i8 [[LB12]], i32 0
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i8> [[TMP4]], i8 [[LA11]], i32 1
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i8> [[TMP5]], i8 [[L910]], i32 2
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i8> [[TMP6]], i8 [[L89]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP7]], ptr [[PTR0]], align 4
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i8> poison, i8 [[L78]], i32 0
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i8> [[TMP8]], i8 [[L67]], i32 1
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i8> [[TMP9]], i8 [[L56]], i32 2
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i8> [[TMP10]], i8 [[L45]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP11]], ptr [[PTR4]], align 4
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i8> poison, i8 [[L34]], i32 0
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> [[TMP12]], i8 [[L23]], i32 1
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i8> [[TMP13]], i8 [[L12]], i32 2
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[L01]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP15]], ptr [[PTR8]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b8>, ptr [[PTR0]], align 4
+; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x b8> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[L01]] to i8
+; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x b8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[L12]] to i8
+; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x b8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b8 [[L23]] to i8
+; CHECK-NEXT: [[L34:%.*]] = extractelement <4 x b8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b8 [[L34]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = load <4 x b8>, ptr [[PTR4]], align 4
+; CHECK-NEXT: [[L45:%.*]] = extractelement <4 x b8> [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b8 [[L45]] to i8
+; CHECK-NEXT: [[L56:%.*]] = extractelement <4 x b8> [[TMP6]], i32 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b8 [[L56]] to i8
+; CHECK-NEXT: [[L67:%.*]] = extractelement <4 x b8> [[TMP6]], i32 2
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast b8 [[L67]] to i8
+; CHECK-NEXT: [[L78:%.*]] = extractelement <4 x b8> [[TMP6]], i32 3
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast b8 [[L78]] to i8
+; CHECK-NEXT: [[TMP11:%.*]] = load <4 x b8>, ptr [[PTR8]], align 4
+; CHECK-NEXT: [[L89:%.*]] = extractelement <4 x b8> [[TMP11]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast b8 [[L89]] to i8
+; CHECK-NEXT: [[L910:%.*]] = extractelement <4 x b8> [[TMP11]], i32 1
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast b8 [[L910]] to i8
+; CHECK-NEXT: [[LA11:%.*]] = extractelement <4 x b8> [[TMP11]], i32 2
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast b8 [[LA11]] to i8
+; CHECK-NEXT: [[LB12:%.*]] = extractelement <4 x b8> [[TMP11]], i32 3
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast b8 [[LB12]] to i8
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i8 [[TMP15]] to b8
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x b8> poison, b8 [[TMP16]], i32 0
+; CHECK-NEXT: [[TMP18:%.*]] = bitcast i8 [[TMP14]] to b8
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x b8> [[TMP17]], b8 [[TMP18]], i32 1
+; CHECK-NEXT: [[TMP20:%.*]] = bitcast i8 [[TMP13]] to b8
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x b8> [[TMP19]], b8 [[TMP20]], i32 2
+; CHECK-NEXT: [[TMP22:%.*]] = bitcast i8 [[TMP12]] to b8
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x b8> [[TMP21]], b8 [[TMP22]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP23]], ptr [[PTR0]], align 4
+; CHECK-NEXT: [[TMP24:%.*]] = bitcast i8 [[TMP10]] to b8
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x b8> poison, b8 [[TMP24]], i32 0
+; CHECK-NEXT: [[TMP26:%.*]] = bitcast i8 [[TMP9]] to b8
+; CHECK-NEXT: [[TMP27:%.*]] = insertelement <4 x b8> [[TMP25]], b8 [[TMP26]], i32 1
+; CHECK-NEXT: [[TMP28:%.*]] = bitcast i8 [[TMP8]] to b8
+; CHECK-NEXT: [[TMP29:%.*]] = insertelement <4 x b8> [[TMP27]], b8 [[TMP28]], i32 2
+; CHECK-NEXT: [[TMP30:%.*]] = bitcast i8 [[TMP7]] to b8
+; CHECK-NEXT: [[TMP31:%.*]] = insertelement <4 x b8> [[TMP29]], b8 [[TMP30]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP31]], ptr [[PTR4]], align 4
+; CHECK-NEXT: [[TMP32:%.*]] = bitcast i8 [[TMP5]] to b8
+; CHECK-NEXT: [[TMP33:%.*]] = insertelement <4 x b8> poison, b8 [[TMP32]], i32 0
+; CHECK-NEXT: [[TMP34:%.*]] = bitcast i8 [[TMP4]] to b8
+; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x b8> [[TMP33]], b8 [[TMP34]], i32 1
+; CHECK-NEXT: [[TMP36:%.*]] = bitcast i8 [[TMP3]] to b8
+; CHECK-NEXT: [[TMP37:%.*]] = insertelement <4 x b8> [[TMP35]], b8 [[TMP36]], i32 2
+; CHECK-NEXT: [[TMP38:%.*]] = bitcast i8 [[TMP2]] to b8
+; CHECK-NEXT: [[TMP39:%.*]] = insertelement <4 x b8> [[TMP37]], b8 [[TMP38]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP39]], ptr [[PTR8]], align 4
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i64 0
@@ -155,46 +188,78 @@ define void @int8x16a4(ptr nocapture align 4 %ptr) {
; CHECK-NEXT: [[PTR4:%.*]] = getelementptr i8, ptr [[PTR]], i64 4
; CHECK-NEXT: [[PTR8:%.*]] = getelementptr i8, ptr [[PTR]], i64 8
; CHECK-NEXT: [[PTRC:%.*]] = getelementptr i8, ptr [[PTR]], i64 12
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[PTR0]], align 4
-; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[L34:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[PTR4]], align 4
-; CHECK-NEXT: [[L45:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
-; CHECK-NEXT: [[L56:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
-; CHECK-NEXT: [[L67:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
-; CHECK-NEXT: [[L78:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
-; CHECK-NEXT: [[TMP3:%.*]] = load <4 x i8>, ptr [[PTR8]], align 4
-; CHECK-NEXT: [[L89:%.*]] = extractelement <4 x i8> [[TMP3]], i32 0
-; CHECK-NEXT: [[L910:%.*]] = extractelement <4 x i8> [[TMP3]], i32 1
-; CHECK-NEXT: [[LA11:%.*]] = extractelement <4 x i8> [[TMP3]], i32 2
-; CHECK-NEXT: [[LB12:%.*]] = extractelement <4 x i8> [[TMP3]], i32 3
-; CHECK-NEXT: [[TMP4:%.*]] = load <4 x i8>, ptr [[PTRC]], align 4
-; CHECK-NEXT: [[LC13:%.*]] = extractelement <4 x i8> [[TMP4]], i32 0
-; CHECK-NEXT: [[LD14:%.*]] = extractelement <4 x i8> [[TMP4]], i32 1
-; CHECK-NEXT: [[LE15:%.*]] = extractelement <4 x i8> [[TMP4]], i32 2
-; CHECK-NEXT: [[LF16:%.*]] = extractelement <4 x i8> [[TMP4]], i32 3
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i8> poison, i8 [[LF16]], i32 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i8> [[TMP5]], i8 [[LE15]], i32 1
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i8> [[TMP6]], i8 [[LD14]], i32 2
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i8> [[TMP7]], i8 [[LC13]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP8]], ptr [[PTRC]], align 4
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i8> poison, i8 [[LB12]], i32 0
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x i8> [[TMP9]], i8 [[LA11]], i32 1
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i8> [[TMP10]], i8 [[L910]], i32 2
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i8> [[TMP11]], i8 [[L89]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP12]], ptr [[PTR0]], align 4
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> poison, i8 [[L78]], i32 0
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i8> [[TMP13]], i8 [[L67]], i32 1
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x i8> [[TMP14]], i8 [[L56]], i32 2
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x i8> [[TMP15]], i8 [[L45]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP16]], ptr [[PTR4]], align 4
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i8> poison, i8 [[L34]], i32 0
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i8> [[TMP17]], i8 [[L23]], i32 1
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> [[TMP18]], i8 [[L12]], i32 2
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[L01]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP20]], ptr [[PTR8]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b8>, ptr [[PTR0]], align 4
+; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x b8> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[L01]] to i8
+; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x b8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[L12]] to i8
+; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x b8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b8 [[L23]] to i8
+; CHECK-NEXT: [[L34:%.*]] = extractelement <4 x b8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b8 [[L34]] to i8
+; CHECK-NEXT: [[TMP6:%.*]] = load <4 x b8>, ptr [[PTR4]], align 4
+; CHECK-NEXT: [[L45:%.*]] = extractelement <4 x b8> [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b8 [[L45]] to i8
+; CHECK-NEXT: [[L56:%.*]] = extractelement <4 x b8> [[TMP6]], i32 1
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b8 [[L56]] to i8
+; CHECK-NEXT: [[L67:%.*]] = extractelement <4 x b8> [[TMP6]], i32 2
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast b8 [[L67]] to i8
+; CHECK-NEXT: [[L78:%.*]] = extractelement <4 x b8> [[TMP6]], i32 3
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast b8 [[L78]] to i8
+; CHECK-NEXT: [[TMP11:%.*]] = load <4 x b8>, ptr [[PTR8]], align 4
+; CHECK-NEXT: [[L89:%.*]] = extractelement <4 x b8> [[TMP11]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast b8 [[L89]] to i8
+; CHECK-NEXT: [[L910:%.*]] = extractelement <4 x b8> [[TMP11]], i32 1
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast b8 [[L910]] to i8
+; CHECK-NEXT: [[LA11:%.*]] = extractelement <4 x b8> [[TMP11]], i32 2
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast b8 [[LA11]] to i8
+; CHECK-NEXT: [[LB12:%.*]] = extractelement <4 x b8> [[TMP11]], i32 3
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast b8 [[LB12]] to i8
+; CHECK-NEXT: [[TMP16:%.*]] = load <4 x b8>, ptr [[PTRC]], align 4
+; CHECK-NEXT: [[LC13:%.*]] = extractelement <4 x b8> [[TMP16]], i32 0
+; CHECK-NEXT: [[TMP17:%.*]] = bitcast b8 [[LC13]] to i8
+; CHECK-NEXT: [[LD14:%.*]] = extractelement <4 x b8> [[TMP16]], i32 1
+; CHECK-NEXT: [[TMP18:%.*]] = bitcast b8 [[LD14]] to i8
+; CHECK-NEXT: [[LE15:%.*]] = extractelement <4 x b8> [[TMP16]], i32 2
+; CHECK-NEXT: [[TMP19:%.*]] = bitcast b8 [[LE15]] to i8
+; CHECK-NEXT: [[LF16:%.*]] = extractelement <4 x b8> [[TMP16]], i32 3
+; CHECK-NEXT: [[TMP20:%.*]] = bitcast b8 [[LF16]] to i8
+; CHECK-NEXT: [[TMP21:%.*]] = bitcast i8 [[TMP20]] to b8
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x b8> poison, b8 [[TMP21]], i32 0
+; CHECK-NEXT: [[TMP23:%.*]] = bitcast i8 [[TMP19]] to b8
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x b8> [[TMP22]], b8 [[TMP23]], i32 1
+; CHECK-NEXT: [[TMP25:%.*]] = bitcast i8 [[TMP18]] to b8
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <4 x b8> [[TMP24]], b8 [[TMP25]], i32 2
+; CHECK-NEXT: [[TMP27:%.*]] = bitcast i8 [[TMP17]] to b8
+; CHECK-NEXT: [[TMP28:%.*]] = insertelement <4 x b8> [[TMP26]], b8 [[TMP27]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP28]], ptr [[PTRC]], align 4
+; CHECK-NEXT: [[TMP29:%.*]] = bitcast i8 [[TMP15]] to b8
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x b8> poison, b8 [[TMP29]], i32 0
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast i8 [[TMP14]] to b8
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x b8> [[TMP30]], b8 [[TMP31]], i32 1
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i8 [[TMP13]] to b8
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <4 x b8> [[TMP32]], b8 [[TMP33]], i32 2
+; CHECK-NEXT: [[TMP35:%.*]] = bitcast i8 [[TMP12]] to b8
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x b8> [[TMP34]], b8 [[TMP35]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP36]], ptr [[PTR0]], align 4
+; CHECK-NEXT: [[TMP37:%.*]] = bitcast i8 [[TMP10]] to b8
+; CHECK-NEXT: [[TMP38:%.*]] = insertelement <4 x b8> poison, b8 [[TMP37]], i32 0
+; CHECK-NEXT: [[TMP39:%.*]] = bitcast i8 [[TMP9]] to b8
+; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x b8> [[TMP38]], b8 [[TMP39]], i32 1
+; CHECK-NEXT: [[TMP41:%.*]] = bitcast i8 [[TMP8]] to b8
+; CHECK-NEXT: [[TMP42:%.*]] = insertelement <4 x b8> [[TMP40]], b8 [[TMP41]], i32 2
+; CHECK-NEXT: [[TMP43:%.*]] = bitcast i8 [[TMP7]] to b8
+; CHECK-NEXT: [[TMP44:%.*]] = insertelement <4 x b8> [[TMP42]], b8 [[TMP43]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP44]], ptr [[PTR4]], align 4
+; CHECK-NEXT: [[TMP45:%.*]] = bitcast i8 [[TMP5]] to b8
+; CHECK-NEXT: [[TMP46:%.*]] = insertelement <4 x b8> poison, b8 [[TMP45]], i32 0
+; CHECK-NEXT: [[TMP47:%.*]] = bitcast i8 [[TMP4]] to b8
+; CHECK-NEXT: [[TMP48:%.*]] = insertelement <4 x b8> [[TMP46]], b8 [[TMP47]], i32 1
+; CHECK-NEXT: [[TMP49:%.*]] = bitcast i8 [[TMP3]] to b8
+; CHECK-NEXT: [[TMP50:%.*]] = insertelement <4 x b8> [[TMP48]], b8 [[TMP49]], i32 2
+; CHECK-NEXT: [[TMP51:%.*]] = bitcast i8 [[TMP2]] to b8
+; CHECK-NEXT: [[TMP52:%.*]] = insertelement <4 x b8> [[TMP50]], b8 [[TMP51]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP52]], ptr [[PTR8]], align 4
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i64 0
@@ -256,24 +321,40 @@ define void @int8x8a8(ptr nocapture align 8 %ptr) {
; CHECK-LABEL: define void @int8x8a8(
; CHECK-SAME: ptr align 8 captures(none) [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[PTR0]], align 8
-; CHECK-NEXT: [[L01:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[L23:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[L34:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[L45:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
-; CHECK-NEXT: [[L56:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
-; CHECK-NEXT: [[L67:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
-; CHECK-NEXT: [[L78:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
-; CHECK-NEXT: [[TMP2:%.*]] = insertelement <8 x i8> poison, i8 [[L78]], i32 0
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i8> [[TMP2]], i8 [[L67]], i32 1
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i8> [[TMP3]], i8 [[L56]], i32 2
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i8> [[TMP4]], i8 [[L45]], i32 3
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i8> [[TMP5]], i8 [[L34]], i32 4
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x i8> [[TMP6]], i8 [[L23]], i32 5
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[L12]], i32 6
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i8> [[TMP8]], i8 [[L01]], i32 7
-; CHECK-NEXT: store <8 x i8> [[TMP9]], ptr [[PTR0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x b8>, ptr [[PTR0]], align 8
+; CHECK-NEXT: [[L01:%.*]] = extractelement <8 x b8> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[L01]] to i8
+; CHECK-NEXT: [[L12:%.*]] = extractelement <8 x b8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[L12]] to i8
+; CHECK-NEXT: [[L23:%.*]] = extractelement <8 x b8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b8 [[L23]] to i8
+; CHECK-NEXT: [[L34:%.*]] = extractelement <8 x b8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b8 [[L34]] to i8
+; CHECK-NEXT: [[L45:%.*]] = extractelement <8 x b8> [[TMP1]], i32 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b8 [[L45]] to i8
+; CHECK-NEXT: [[L56:%.*]] = extractelement <8 x b8> [[TMP1]], i32 5
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b8 [[L56]] to i8
+; CHECK-NEXT: [[L67:%.*]] = extractelement <8 x b8> [[TMP1]], i32 6
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b8 [[L67]] to i8
+; CHECK-NEXT: [[L78:%.*]] = extractelement <8 x b8> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast b8 [[L78]] to i8
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast i8 [[TMP9]] to b8
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x b8> poison, b8 [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast i8 [[TMP8]] to b8
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x b8> [[TMP11]], b8 [[TMP12]], i32 1
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast i8 [[TMP7]] to b8
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x b8> [[TMP13]], b8 [[TMP14]], i32 2
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast i8 [[TMP6]] to b8
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x b8> [[TMP15]], b8 [[TMP16]], i32 3
+; CHECK-NEXT: [[TMP18:%.*]] = bitcast i8 [[TMP5]] to b8
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <8 x b8> [[TMP17]], b8 [[TMP18]], i32 4
+; CHECK-NEXT: [[TMP20:%.*]] = bitcast i8 [[TMP4]] to b8
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <8 x b8> [[TMP19]], b8 [[TMP20]], i32 5
+; CHECK-NEXT: [[TMP22:%.*]] = bitcast i8 [[TMP3]] to b8
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <8 x b8> [[TMP21]], b8 [[TMP22]], i32 6
+; CHECK-NEXT: [[TMP24:%.*]] = bitcast i8 [[TMP2]] to b8
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <8 x b8> [[TMP23]], b8 [[TMP24]], i32 7
+; CHECK-NEXT: store <8 x b8> [[TMP25]], ptr [[PTR0]], align 8
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i64 0
@@ -312,34 +393,58 @@ define void @int8x12a8(ptr nocapture align 8 %ptr) {
; CHECK-SAME: ptr align 8 captures(none) [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
; CHECK-NEXT: [[PTR8:%.*]] = getelementptr i8, ptr [[PTR]], i64 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[PTR0]], align 8
-; CHECK-NEXT: [[L01:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[L23:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[L34:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[L45:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
-; CHECK-NEXT: [[L56:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
-; CHECK-NEXT: [[L67:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
-; CHECK-NEXT: [[L78:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x i8>, ptr [[PTR8]], align 8
-; CHECK-NEXT: [[L89:%.*]] = extractelement <4 x i8> [[TMP2]], i32 0
-; CHECK-NEXT: [[L910:%.*]] = extractelement <4 x i8> [[TMP2]], i32 1
-; CHECK-NEXT: [[LA11:%.*]] = extractelement <4 x i8> [[TMP2]], i32 2
-; CHECK-NEXT: [[LB12:%.*]] = extractelement <4 x i8> [[TMP2]], i32 3
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i8> poison, i8 [[LB12]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i8> [[TMP3]], i8 [[LA11]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i8> [[TMP4]], i8 [[L910]], i32 2
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i8> [[TMP5]], i8 [[L89]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x i8> [[TMP6]], i8 [[L78]], i32 4
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[L67]], i32 5
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i8> [[TMP8]], i8 [[L56]], i32 6
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x i8> [[TMP9]], i8 [[L45]], i32 7
-; CHECK-NEXT: store <8 x i8> [[TMP10]], ptr [[PTR0]], align 8
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i8> poison, i8 [[L34]], i32 0
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i8> [[TMP11]], i8 [[L23]], i32 1
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i8> [[TMP12]], i8 [[L12]], i32 2
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x i8> [[TMP13]], i8 [[L01]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP14]], ptr [[PTR8]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x b8>, ptr [[PTR0]], align 8
+; CHECK-NEXT: [[L01:%.*]] = extractelement <8 x b8> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[L01]] to i8
+; CHECK-NEXT: [[L12:%.*]] = extractelement <8 x b8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[L12]] to i8
+; CHECK-NEXT: [[L23:%.*]] = extractelement <8 x b8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b8 [[L23]] to i8
+; CHECK-NEXT: [[L34:%.*]] = extractelement <8 x b8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b8 [[L34]] to i8
+; CHECK-NEXT: [[L45:%.*]] = extractelement <8 x b8> [[TMP1]], i32 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b8 [[L45]] to i8
+; CHECK-NEXT: [[L56:%.*]] = extractelement <8 x b8> [[TMP1]], i32 5
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b8 [[L56]] to i8
+; CHECK-NEXT: [[L67:%.*]] = extractelement <8 x b8> [[TMP1]], i32 6
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b8 [[L67]] to i8
+; CHECK-NEXT: [[L78:%.*]] = extractelement <8 x b8> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast b8 [[L78]] to i8
+; CHECK-NEXT: [[TMP10:%.*]] = load <4 x b8>, ptr [[PTR8]], align 8
+; CHECK-NEXT: [[L89:%.*]] = extractelement <4 x b8> [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast b8 [[L89]] to i8
+; CHECK-NEXT: [[L910:%.*]] = extractelement <4 x b8> [[TMP10]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast b8 [[L910]] to i8
+; CHECK-NEXT: [[LA11:%.*]] = extractelement <4 x b8> [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast b8 [[LA11]] to i8
+; CHECK-NEXT: [[LB12:%.*]] = extractelement <4 x b8> [[TMP10]], i32 3
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast b8 [[LB12]] to i8
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast i8 [[TMP14]] to b8
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x b8> poison, b8 [[TMP15]], i32 0
+; CHECK-NEXT: [[TMP17:%.*]] = bitcast i8 [[TMP13]] to b8
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <8 x b8> [[TMP16]], b8 [[TMP17]], i32 1
+; CHECK-NEXT: [[TMP19:%.*]] = bitcast i8 [[TMP12]] to b8
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <8 x b8> [[TMP18]], b8 [[TMP19]], i32 2
+; CHECK-NEXT: [[TMP21:%.*]] = bitcast i8 [[TMP11]] to b8
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <8 x b8> [[TMP20]], b8 [[TMP21]], i32 3
+; CHECK-NEXT: [[TMP23:%.*]] = bitcast i8 [[TMP9]] to b8
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <8 x b8> [[TMP22]], b8 [[TMP23]], i32 4
+; CHECK-NEXT: [[TMP25:%.*]] = bitcast i8 [[TMP8]] to b8
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <8 x b8> [[TMP24]], b8 [[TMP25]], i32 5
+; CHECK-NEXT: [[TMP27:%.*]] = bitcast i8 [[TMP7]] to b8
+; CHECK-NEXT: [[TMP28:%.*]] = insertelement <8 x b8> [[TMP26]], b8 [[TMP27]], i32 6
+; CHECK-NEXT: [[TMP29:%.*]] = bitcast i8 [[TMP6]] to b8
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <8 x b8> [[TMP28]], b8 [[TMP29]], i32 7
+; CHECK-NEXT: store <8 x b8> [[TMP30]], ptr [[PTR0]], align 8
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast i8 [[TMP5]] to b8
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x b8> poison, b8 [[TMP31]], i32 0
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i8 [[TMP4]] to b8
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <4 x b8> [[TMP32]], b8 [[TMP33]], i32 1
+; CHECK-NEXT: [[TMP35:%.*]] = bitcast i8 [[TMP3]] to b8
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x b8> [[TMP34]], b8 [[TMP35]], i32 2
+; CHECK-NEXT: [[TMP37:%.*]] = bitcast i8 [[TMP2]] to b8
+; CHECK-NEXT: [[TMP38:%.*]] = insertelement <4 x b8> [[TMP36]], b8 [[TMP37]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP38]], ptr [[PTR8]], align 8
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i64 0
@@ -391,42 +496,74 @@ define void @int8x16a8(ptr nocapture align 8 %ptr) {
; CHECK-SAME: ptr align 8 captures(none) [[PTR:%.*]]) {
; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
; CHECK-NEXT: [[PTR8:%.*]] = getelementptr i8, ptr [[PTR]], i64 8
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i8>, ptr [[PTR0]], align 8
-; CHECK-NEXT: [[L01:%.*]] = extractelement <8 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <8 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[L23:%.*]] = extractelement <8 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[L34:%.*]] = extractelement <8 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[L45:%.*]] = extractelement <8 x i8> [[TMP1]], i32 4
-; CHECK-NEXT: [[L56:%.*]] = extractelement <8 x i8> [[TMP1]], i32 5
-; CHECK-NEXT: [[L67:%.*]] = extractelement <8 x i8> [[TMP1]], i32 6
-; CHECK-NEXT: [[L78:%.*]] = extractelement <8 x i8> [[TMP1]], i32 7
-; CHECK-NEXT: [[TMP2:%.*]] = load <8 x i8>, ptr [[PTR8]], align 8
-; CHECK-NEXT: [[L89:%.*]] = extractelement <8 x i8> [[TMP2]], i32 0
-; CHECK-NEXT: [[L910:%.*]] = extractelement <8 x i8> [[TMP2]], i32 1
-; CHECK-NEXT: [[LA11:%.*]] = extractelement <8 x i8> [[TMP2]], i32 2
-; CHECK-NEXT: [[LB12:%.*]] = extractelement <8 x i8> [[TMP2]], i32 3
-; CHECK-NEXT: [[LC13:%.*]] = extractelement <8 x i8> [[TMP2]], i32 4
-; CHECK-NEXT: [[LD14:%.*]] = extractelement <8 x i8> [[TMP2]], i32 5
-; CHECK-NEXT: [[LE15:%.*]] = extractelement <8 x i8> [[TMP2]], i32 6
-; CHECK-NEXT: [[LF16:%.*]] = extractelement <8 x i8> [[TMP2]], i32 7
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <8 x i8> poison, i8 [[LF16]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <8 x i8> [[TMP3]], i8 [[LE15]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <8 x i8> [[TMP4]], i8 [[LD14]], i32 2
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <8 x i8> [[TMP5]], i8 [[LC13]], i32 3
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x i8> [[TMP6]], i8 [[LB12]], i32 4
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x i8> [[TMP7]], i8 [[LA11]], i32 5
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x i8> [[TMP8]], i8 [[L910]], i32 6
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x i8> [[TMP9]], i8 [[L89]], i32 7
-; CHECK-NEXT: store <8 x i8> [[TMP10]], ptr [[PTR0]], align 8
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x i8> poison, i8 [[L78]], i32 0
-; CHECK-NEXT: [[TMP12:%.*]] = insertelement <8 x i8> [[TMP11]], i8 [[L67]], i32 1
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x i8> [[TMP12]], i8 [[L56]], i32 2
-; CHECK-NEXT: [[TMP14:%.*]] = insertelement <8 x i8> [[TMP13]], i8 [[L45]], i32 3
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x i8> [[TMP14]], i8 [[L34]], i32 4
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x i8> [[TMP15]], i8 [[L23]], i32 5
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x i8> [[TMP16]], i8 [[L12]], i32 6
-; CHECK-NEXT: [[TMP18:%.*]] = insertelement <8 x i8> [[TMP17]], i8 [[L01]], i32 7
-; CHECK-NEXT: store <8 x i8> [[TMP18]], ptr [[PTR8]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <8 x b8>, ptr [[PTR0]], align 8
+; CHECK-NEXT: [[L01:%.*]] = extractelement <8 x b8> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[L01]] to i8
+; CHECK-NEXT: [[L12:%.*]] = extractelement <8 x b8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[L12]] to i8
+; CHECK-NEXT: [[L23:%.*]] = extractelement <8 x b8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b8 [[L23]] to i8
+; CHECK-NEXT: [[L34:%.*]] = extractelement <8 x b8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b8 [[L34]] to i8
+; CHECK-NEXT: [[L45:%.*]] = extractelement <8 x b8> [[TMP1]], i32 4
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b8 [[L45]] to i8
+; CHECK-NEXT: [[L56:%.*]] = extractelement <8 x b8> [[TMP1]], i32 5
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b8 [[L56]] to i8
+; CHECK-NEXT: [[L67:%.*]] = extractelement <8 x b8> [[TMP1]], i32 6
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast b8 [[L67]] to i8
+; CHECK-NEXT: [[L78:%.*]] = extractelement <8 x b8> [[TMP1]], i32 7
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast b8 [[L78]] to i8
+; CHECK-NEXT: [[TMP10:%.*]] = load <8 x b8>, ptr [[PTR8]], align 8
+; CHECK-NEXT: [[L89:%.*]] = extractelement <8 x b8> [[TMP10]], i32 0
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast b8 [[L89]] to i8
+; CHECK-NEXT: [[L910:%.*]] = extractelement <8 x b8> [[TMP10]], i32 1
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast b8 [[L910]] to i8
+; CHECK-NEXT: [[LA11:%.*]] = extractelement <8 x b8> [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP13:%.*]] = bitcast b8 [[LA11]] to i8
+; CHECK-NEXT: [[LB12:%.*]] = extractelement <8 x b8> [[TMP10]], i32 3
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast b8 [[LB12]] to i8
+; CHECK-NEXT: [[LC13:%.*]] = extractelement <8 x b8> [[TMP10]], i32 4
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast b8 [[LC13]] to i8
+; CHECK-NEXT: [[LD14:%.*]] = extractelement <8 x b8> [[TMP10]], i32 5
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast b8 [[LD14]] to i8
+; CHECK-NEXT: [[LE15:%.*]] = extractelement <8 x b8> [[TMP10]], i32 6
+; CHECK-NEXT: [[TMP17:%.*]] = bitcast b8 [[LE15]] to i8
+; CHECK-NEXT: [[LF16:%.*]] = extractelement <8 x b8> [[TMP10]], i32 7
+; CHECK-NEXT: [[TMP18:%.*]] = bitcast b8 [[LF16]] to i8
+; CHECK-NEXT: [[TMP19:%.*]] = bitcast i8 [[TMP18]] to b8
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <8 x b8> poison, b8 [[TMP19]], i32 0
+; CHECK-NEXT: [[TMP21:%.*]] = bitcast i8 [[TMP17]] to b8
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <8 x b8> [[TMP20]], b8 [[TMP21]], i32 1
+; CHECK-NEXT: [[TMP23:%.*]] = bitcast i8 [[TMP16]] to b8
+; CHECK-NEXT: [[TMP24:%.*]] = insertelement <8 x b8> [[TMP22]], b8 [[TMP23]], i32 2
+; CHECK-NEXT: [[TMP25:%.*]] = bitcast i8 [[TMP15]] to b8
+; CHECK-NEXT: [[TMP26:%.*]] = insertelement <8 x b8> [[TMP24]], b8 [[TMP25]], i32 3
+; CHECK-NEXT: [[TMP27:%.*]] = bitcast i8 [[TMP14]] to b8
+; CHECK-NEXT: [[TMP28:%.*]] = insertelement <8 x b8> [[TMP26]], b8 [[TMP27]], i32 4
+; CHECK-NEXT: [[TMP29:%.*]] = bitcast i8 [[TMP13]] to b8
+; CHECK-NEXT: [[TMP30:%.*]] = insertelement <8 x b8> [[TMP28]], b8 [[TMP29]], i32 5
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast i8 [[TMP12]] to b8
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <8 x b8> [[TMP30]], b8 [[TMP31]], i32 6
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i8 [[TMP11]] to b8
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <8 x b8> [[TMP32]], b8 [[TMP33]], i32 7
+; CHECK-NEXT: store <8 x b8> [[TMP34]], ptr [[PTR0]], align 8
+; CHECK-NEXT: [[TMP35:%.*]] = bitcast i8 [[TMP9]] to b8
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <8 x b8> poison, b8 [[TMP35]], i32 0
+; CHECK-NEXT: [[TMP37:%.*]] = bitcast i8 [[TMP8]] to b8
+; CHECK-NEXT: [[TMP38:%.*]] = insertelement <8 x b8> [[TMP36]], b8 [[TMP37]], i32 1
+; CHECK-NEXT: [[TMP39:%.*]] = bitcast i8 [[TMP7]] to b8
+; CHECK-NEXT: [[TMP40:%.*]] = insertelement <8 x b8> [[TMP38]], b8 [[TMP39]], i32 2
+; CHECK-NEXT: [[TMP41:%.*]] = bitcast i8 [[TMP6]] to b8
+; CHECK-NEXT: [[TMP42:%.*]] = insertelement <8 x b8> [[TMP40]], b8 [[TMP41]], i32 3
+; CHECK-NEXT: [[TMP43:%.*]] = bitcast i8 [[TMP5]] to b8
+; CHECK-NEXT: [[TMP44:%.*]] = insertelement <8 x b8> [[TMP42]], b8 [[TMP43]], i32 4
+; CHECK-NEXT: [[TMP45:%.*]] = bitcast i8 [[TMP4]] to b8
+; CHECK-NEXT: [[TMP46:%.*]] = insertelement <8 x b8> [[TMP44]], b8 [[TMP45]], i32 5
+; CHECK-NEXT: [[TMP47:%.*]] = bitcast i8 [[TMP3]] to b8
+; CHECK-NEXT: [[TMP48:%.*]] = insertelement <8 x b8> [[TMP46]], b8 [[TMP47]], i32 6
+; CHECK-NEXT: [[TMP49:%.*]] = bitcast i8 [[TMP2]] to b8
+; CHECK-NEXT: [[TMP50:%.*]] = insertelement <8 x b8> [[TMP48]], b8 [[TMP49]], i32 7
+; CHECK-NEXT: store <8 x b8> [[TMP50]], ptr [[PTR8]], align 8
; CHECK-NEXT: ret void
;
%ptr0 = getelementptr i8, ptr %ptr, i64 0
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_vectors.ll b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_vectors.ll
index 91242f6e50b4e..aef7b007b6073 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_vectors.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/NVPTX/vectorize_vectors.ll
@@ -1,9 +1,18 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=nvptx64-nvidia-cuda -passes=load-store-vectorizer -S -o - %s | FileCheck %s
-; CHECK-LABEL: @int8x3Plus1
-; CHECK: load <4 x i8>
-; CHECK: store <4 x i8>
define void @int8x3Plus1(ptr nocapture align 4 %ptr) {
+; CHECK-LABEL: define void @int8x3Plus1(
+; CHECK-SAME: ptr align 4 captures(none) [[PTR:%.*]]) {
+; CHECK-NEXT: [[PTR0:%.*]] = getelementptr i8, ptr [[PTR]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b8>, ptr [[PTR0]], align 4
+; CHECK-NEXT: [[L01:%.*]] = shufflevector <4 x b8> [[TMP1]], <4 x b8> poison, <3 x i32> <i32 0, i32 1, i32 2>
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast <3 x b8> [[L01]] to <3 x i8>
+; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x b8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[L12]] to i8
+; CHECK-NEXT: store <4 x b8> zeroinitializer, ptr [[PTR0]], align 4
+; CHECK-NEXT: ret void
+;
%ptr0 = getelementptr i8, ptr %ptr, i64 0
%ptr3 = getelementptr i8, ptr %ptr, i64 3
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll
index a56efe8dd3f35..7606bb99dec19 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/codegenprepare-produced-address-math.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes='require<profile-summary>,function(codegenprepare)' -passes=load-store-vectorizer %s -S -o - | FileCheck %s
; RUN: opt -passes=load-store-vectorizer %s -S -o - | FileCheck %s
; RUN: opt -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' %s -S -o - | FileCheck %s
@@ -10,10 +11,25 @@ target triple = "x86_64--"
; Function Attrs: convergent nounwind
define void @test(i32 %base) #0 {
-; CHECK-LABEL: @test(
-; CHECK-NOT: load i32
-; CHECK: load <2 x i32>
-; CHECK-NOT: load i32
+; CHECK-LABEL: define void @test(
+; CHECK-SAME: i32 [[BASE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MUL331:%.*]] = and i32 [[BASE]], -4
+; CHECK-NEXT: [[ADD350_4:%.*]] = add i32 4, [[MUL331]]
+; CHECK-NEXT: [[IDX351_4:%.*]] = zext i32 [[ADD350_4]] to i64
+; CHECK-NEXT: [[ARRAYIDX352_4:%.*]] = getelementptr inbounds { [[UNION:%.*]], [2000 x i8] }, ptr @global_pointer, i64 0, i32 0, i32 0, i32 1, i64 0, i64 0, i64 0, i64 [[IDX351_4]]
+; CHECK-NEXT: [[CND:%.*]] = icmp ult i32 [[BASE]], 1000
+; CHECK-NEXT: br i1 [[CND]], label %[[LOADS:.*]], label %[[EXIT:.*]]
+; CHECK: [[LOADS]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr [[ARRAYIDX352_4]], align 4, !tbaa [[FLOAT_TBAA0:![0-9]+]]
+; CHECK-NEXT: [[TMP297_41:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[TMP297_41]] to i32
+; CHECK-NEXT: [[TMP297_52:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[TMP297_52]] to i32
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
entry:
%mul331 = and i32 %base, -4
%add350.4 = add i32 4, %mul331
@@ -39,10 +55,27 @@ exit:
; Function Attrs: convergent nounwind
define void @test.codegenprepared(i32 %base) #0 {
-; CHECK-LABEL: @test.codegenprepared(
-; CHECK-NOT: load i32
-; CHECK: load <2 x i32>
-; CHECK-NOT: load i32
+; CHECK-LABEL: define void @test.codegenprepared(
+; CHECK-SAME: i32 [[BASE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[MUL331:%.*]] = and i32 [[BASE]], -4
+; CHECK-NEXT: [[ADD350_4:%.*]] = add i32 4, [[MUL331]]
+; CHECK-NEXT: [[IDX351_4:%.*]] = zext i32 [[ADD350_4]] to i64
+; CHECK-NEXT: [[CND:%.*]] = icmp ult i32 [[BASE]], 1000
+; CHECK-NEXT: br i1 [[CND]], label %[[LOADS:.*]], label %[[EXIT:.*]]
+; CHECK: [[LOADS]]:
+; CHECK-NEXT: [[SUNKADDR:%.*]] = mul i64 [[IDX351_4]], 4
+; CHECK-NEXT: [[SUNKADDR1:%.*]] = getelementptr inbounds i8, ptr @global_pointer, i64 [[SUNKADDR]]
+; CHECK-NEXT: [[SUNKADDR2:%.*]] = getelementptr inbounds i8, ptr [[SUNKADDR1]], i64 4096
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr [[SUNKADDR2]], align 4, !tbaa [[FLOAT_TBAA0]]
+; CHECK-NEXT: [[TMP297_41:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[TMP297_41]] to i32
+; CHECK-NEXT: [[TMP297_52:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[TMP297_52]] to i32
+; CHECK-NEXT: br label %[[EXIT]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
entry:
%mul331 = and i32 %base, -4
%add350.4 = add i32 4, %mul331
@@ -73,3 +106,9 @@ attributes #0 = { convergent nounwind }
!1 = !{!"float", !2, i64 0}
!2 = !{!"omnipotent char", !3, i64 0}
!3 = !{!"Simple C++ TBAA"}
+;.
+; CHECK: [[FLOAT_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"float", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"omnipotent char", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"Simple C++ TBAA"}
+;.
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/compare-scev-by-complexity.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/compare-scev-by-complexity.ll
index 4051ed4554be7..2bda878aa3c43 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/compare-scev-by-complexity.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/compare-scev-by-complexity.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -passes=load-store-vectorizer %s -S | FileCheck %s
; RUN: opt -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' %s -S | FileCheck %s
@@ -19,9 +20,52 @@ target triple = "x86_64--"
; Function Attrs: nounwind
define void @main() local_unnamed_addr #0 {
-; CHECK-LABEL: @main()
-; CHECK: = load <2 x float>, ptr %preheader.load0.address
-; CHECK-LABEL: for.body23:
+; CHECK-LABEL: define void @main(
+; CHECK-SAME: ) local_unnamed_addr #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP:%.*]] = load i32, ptr @global_value0, align 4, !range [[RNG0:![0-9]+]]
+; CHECK-NEXT: [[TMP2:%.*]] = load i32, ptr @global_value1, align 4
+; CHECK-NEXT: [[AND_I_I:%.*]] = and i32 [[TMP2]], 2
+; CHECK-NEXT: [[ADD_NUW_NSW_I_I:%.*]] = add nuw nsw i32 [[AND_I_I]], 0
+; CHECK-NEXT: [[MUL_I_I:%.*]] = shl nuw nsw i32 [[ADD_NUW_NSW_I_I]], 1
+; CHECK-NEXT: [[AND6_I_I:%.*]] = and i32 [[TMP2]], 3
+; CHECK-NEXT: [[AND9_I_I:%.*]] = and i32 [[TMP2]], 4
+; CHECK-NEXT: [[ADD_NUW_NSW10_I_I:%.*]] = add nuw nsw i32 [[AND6_I_I]], [[AND9_I_I]]
+; CHECK-NEXT: [[CONV3_I42_I:%.*]] = add nuw nsw i32 [[MUL_I_I]], 1
+; CHECK-NEXT: [[REASS_ADD346_7:%.*]] = add nuw nsw i32 [[ADD_NUW_NSW10_I_I]], 56
+; CHECK-NEXT: [[REASS_MUL347_7:%.*]] = mul nuw nsw i32 [[TMP]], [[REASS_ADD346_7]]
+; CHECK-NEXT: [[ADD7_I_7:%.*]] = add nuw nsw i32 [[REASS_MUL347_7]], 0
+; CHECK-NEXT: [[PREHEADER_ADDRESS0_IDX:%.*]] = add nuw nsw i32 [[ADD7_I_7]], [[MUL_I_I]]
+; CHECK-NEXT: [[PREHEADER_ADDRESS0_IDX_ZEXT:%.*]] = zext i32 [[PREHEADER_ADDRESS0_IDX]] to i64
+; CHECK-NEXT: [[PREHEADER_LOAD0_ADDRESS:%.*]] = getelementptr inbounds float, ptr @other_value, i64 [[PREHEADER_ADDRESS0_IDX_ZEXT]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr [[PREHEADER_LOAD0_ADDRESS]], align 4, !tbaa [[FLOAT_TBAA1:![0-9]+]]
+; CHECK-NEXT: [[PREHEADER_LOAD0_1:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[PREHEADER_LOAD0_1]] to float
+; CHECK-NEXT: [[PREHEADER_LOAD1_2:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[PREHEADER_LOAD1_2]] to float
+; CHECK-NEXT: [[COMMON_ADDRESS_IDX:%.*]] = add nuw nsw i32 [[ADD7_I_7]], [[CONV3_I42_I]]
+; CHECK-NEXT: [[PREHEADER_HEADER_COMMON_ADDRESS_IDX_ZEXT:%.*]] = zext i32 [[COMMON_ADDRESS_IDX]] to i64
+; CHECK-NEXT: br label %[[FOR_BODY23:.*]]
+; CHECK: [[FOR_BODY23]]:
+; CHECK-NEXT: [[LOOP_HEADER_LOAD0_ADDRESS:%.*]] = getelementptr inbounds float, ptr @other_value, i64 [[PREHEADER_HEADER_COMMON_ADDRESS_IDX_ZEXT]]
+; CHECK-NEXT: [[LOOP_HEADER_LOAD0_:%.*]] = load float, ptr [[LOOP_HEADER_LOAD0_ADDRESS]], align 4, !tbaa [[FLOAT_TBAA1]]
+; CHECK-NEXT: [[REASS_MUL343_7:%.*]] = mul nuw nsw i32 [[REASS_ADD346_7]], 72
+; CHECK-NEXT: [[ADD7_I286_7_7:%.*]] = add nuw nsw i32 [[REASS_MUL343_7]], 56
+; CHECK-NEXT: [[ADD9_I288_7_7:%.*]] = add nuw nsw i32 [[ADD7_I286_7_7]], [[MUL_I_I]]
+; CHECK-NEXT: [[LOOP_HEADER_ADDRESS1_IDX:%.*]] = add nuw nsw i32 [[ADD9_I288_7_7]], 1
+; CHECK-NEXT: [[LOOP_HEADER_ADDRESS1_IDX_ZEXT:%.*]] = zext i32 [[LOOP_HEADER_ADDRESS1_IDX]] to i64
+; CHECK-NEXT: [[LOOP_HEADER_LOAD1_ADDRESS:%.*]] = getelementptr inbounds float, ptr @other_value, i64 [[LOOP_HEADER_ADDRESS1_IDX_ZEXT]]
+; CHECK-NEXT: [[LOOP_HEADER_LOAD1_:%.*]] = load float, ptr [[LOOP_HEADER_LOAD1_ADDRESS]], align 4, !tbaa [[FLOAT_TBAA1]]
+; CHECK-NEXT: store float [[TMP1]], ptr @a, align 4, !tbaa [[FLOAT_TBAA1]]
+; CHECK-NEXT: store float [[TMP2]], ptr @b, align 4, !tbaa [[FLOAT_TBAA1]]
+; CHECK-NEXT: store float [[LOOP_HEADER_LOAD0_]], ptr @c, align 4, !tbaa [[FLOAT_TBAA1]]
+; CHECK-NEXT: store float [[LOOP_HEADER_LOAD1_]], ptr @d, align 4, !tbaa [[FLOAT_TBAA1]]
+; CHECK-NEXT: [[LOADED_CND:%.*]] = load i8, ptr @cnd, align 1
+; CHECK-NEXT: [[CONDITION:%.*]] = trunc i8 [[LOADED_CND]] to i1
+; CHECK-NEXT: br i1 [[CONDITION]], label %[[FOR_BODY23]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
entry:
%tmp = load i32, ptr @global_value0, !range !0
%tmp2 = load i32, ptr @global_value1
@@ -74,3 +118,10 @@ attributes #0 = { nounwind }
!2 = !{!"float", !3, i64 0}
!3 = !{!"omnipotent char", !4, i64 0}
!4 = !{!"Simple C++ TBAA"}
+;.
+; CHECK: [[RNG0]] = !{i32 0, i32 65536}
+; CHECK: [[FLOAT_TBAA1]] = !{[[META2:![0-9]+]], [[META2]], i64 0}
+; CHECK: [[META2]] = !{!"float", [[META3:![0-9]+]], i64 0}
+; CHECK: [[META3]] = !{!"omnipotent char", [[META4:![0-9]+]], i64 0}
+; CHECK: [[META4]] = !{!"Simple C++ TBAA"}
+;.
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/correct-order.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/correct-order.ll
index 0fc2ac01e9315..7766520cb2c02 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/correct-order.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/correct-order.ll
@@ -7,13 +7,17 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
define void @correct_order(ptr noalias %ptr) {
; CHECK-LABEL: @correct_order(
; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 1
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[PTR]], align 4
-; CHECK-NEXT: [[L21:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr [[PTR]], align 4
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[NEXT_GEP1]], align 4
-; CHECK-NEXT: [[L33:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
-; CHECK-NEXT: [[L44:%.*]] = extractelement <2 x i32> [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[PTR]], align 4
+; CHECK-NEXT: [[L21:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[L21]] to i32
+; CHECK-NEXT: [[L12:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[L12]] to i32
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr [[PTR]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x b32>, ptr [[NEXT_GEP1]], align 4
+; CHECK-NEXT: [[L33:%.*]] = extractelement <2 x b32> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[L33]] to i32
+; CHECK-NEXT: [[L44:%.*]] = extractelement <2 x b32> [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[L44]] to i32
; CHECK-NEXT: ret void
;
%next.gep1 = getelementptr i32, ptr %ptr, i64 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width-inseltpoison.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width-inseltpoison.ll
index a38aacfc3ce0e..3fb60f0aff3e0 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width-inseltpoison.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width-inseltpoison.ll
@@ -1,39 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=load-store-vectorizer -mcpu haswell -S -o - %s | FileCheck --check-prefix=CHECK-HSW %s
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=load-store-vectorizer -mcpu knl -S -o - %s | FileCheck --check-prefix=CHECK-KNL %s
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -mcpu haswell -S -o - %s | FileCheck --check-prefix=CHECK-HSW %s
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -mcpu knl -S -o - %s | FileCheck --check-prefix=CHECK-KNL %s
define <8 x double> @loadwidth_insert_extract(ptr %ptr) {
- %b = getelementptr <2 x double>, ptr %ptr, i32 1
- %c = getelementptr <2 x double>, ptr %ptr, i32 2
- %d = getelementptr <2 x double>, ptr %ptr, i32 3
-; CHECK-HSW: load <4 x double>
-; CHECK-HSW: load <4 x double>
-; CHECK-HSW-NOT: load
-; CHECK-KNL: load <8 x double>
-; CHECK-KNL-NOT: load
- %la = load <2 x double>, ptr %ptr
- %lb = load <2 x double>, ptr %b
- %lc = load <2 x double>, ptr %c
- %ld = load <2 x double>, ptr %d
- ; Scalarize everything - Explicitly not a shufflevector to test this code
- ; path in the LSV
- %v1 = extractelement <2 x double> %la, i32 0
- %v2 = extractelement <2 x double> %la, i32 1
- %v3 = extractelement <2 x double> %lb, i32 0
- %v4 = extractelement <2 x double> %lb, i32 1
- %v5 = extractelement <2 x double> %lc, i32 0
- %v6 = extractelement <2 x double> %lc, i32 1
- %v7 = extractelement <2 x double> %ld, i32 0
- %v8 = extractelement <2 x double> %ld, i32 1
- ; Make a vector again
- %i1 = insertelement <8 x double> poison, double %v1, i32 0
- %i2 = insertelement <8 x double> %i1, double %v2, i32 1
- %i3 = insertelement <8 x double> %i2, double %v3, i32 2
- %i4 = insertelement <8 x double> %i3, double %v4, i32 3
- %i5 = insertelement <8 x double> %i4, double %v5, i32 4
- %i6 = insertelement <8 x double> %i5, double %v6, i32 5
- %i7 = insertelement <8 x double> %i6, double %v7, i32 6
- %i8 = insertelement <8 x double> %i7, double %v8, i32 7
- ret <8 x double> %i8
+; CHECK-HSW-LABEL: define <8 x double> @loadwidth_insert_extract(
+; CHECK-HSW-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-HSW-NEXT: [[C:%.*]] = getelementptr <2 x double>, ptr [[PTR]], i32 2
+; CHECK-HSW-NEXT: [[TMP1:%.*]] = load <4 x b64>, ptr [[PTR]], align 16
+; CHECK-HSW-NEXT: [[LA1:%.*]] = shufflevector <4 x b64> [[TMP1]], <4 x b64> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-HSW-NEXT: [[TMP2:%.*]] = bitcast <2 x b64> [[LA1]] to <2 x double>
+; CHECK-HSW-NEXT: [[LB2:%.*]] = shufflevector <4 x b64> [[TMP1]], <4 x b64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-HSW-NEXT: [[TMP3:%.*]] = bitcast <2 x b64> [[LB2]] to <2 x double>
+; CHECK-HSW-NEXT: [[TMP4:%.*]] = load <4 x b64>, ptr [[C]], align 16
+; CHECK-HSW-NEXT: [[LC3:%.*]] = shufflevector <4 x b64> [[TMP4]], <4 x b64> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-HSW-NEXT: [[TMP5:%.*]] = bitcast <2 x b64> [[LC3]] to <2 x double>
+; CHECK-HSW-NEXT: [[LD4:%.*]] = shufflevector <4 x b64> [[TMP4]], <4 x b64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-HSW-NEXT: [[TMP6:%.*]] = bitcast <2 x b64> [[LD4]] to <2 x double>
+; CHECK-HSW-NEXT: [[V1:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-HSW-NEXT: [[V2:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
+; CHECK-HSW-NEXT: [[V3:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-HSW-NEXT: [[V4:%.*]] = extractelement <2 x double> [[TMP3]], i32 1
+; CHECK-HSW-NEXT: [[V5:%.*]] = extractelement <2 x double> [[TMP5]], i32 0
+; CHECK-HSW-NEXT: [[V6:%.*]] = extractelement <2 x double> [[TMP5]], i32 1
+; CHECK-HSW-NEXT: [[V7:%.*]] = extractelement <2 x double> [[TMP6]], i32 0
+; CHECK-HSW-NEXT: [[V8:%.*]] = extractelement <2 x double> [[TMP6]], i32 1
+; CHECK-HSW-NEXT: [[I1:%.*]] = insertelement <8 x double> poison, double [[V1]], i32 0
+; CHECK-HSW-NEXT: [[I2:%.*]] = insertelement <8 x double> [[I1]], double [[V2]], i32 1
+; CHECK-HSW-NEXT: [[I3:%.*]] = insertelement <8 x double> [[I2]], double [[V3]], i32 2
+; CHECK-HSW-NEXT: [[I4:%.*]] = insertelement <8 x double> [[I3]], double [[V4]], i32 3
+; CHECK-HSW-NEXT: [[I5:%.*]] = insertelement <8 x double> [[I4]], double [[V5]], i32 4
+; CHECK-HSW-NEXT: [[I6:%.*]] = insertelement <8 x double> [[I5]], double [[V6]], i32 5
+; CHECK-HSW-NEXT: [[I7:%.*]] = insertelement <8 x double> [[I6]], double [[V7]], i32 6
+; CHECK-HSW-NEXT: [[I8:%.*]] = insertelement <8 x double> [[I7]], double [[V8]], i32 7
+; CHECK-HSW-NEXT: ret <8 x double> [[I8]]
+;
+; CHECK-KNL-LABEL: define <8 x double> @loadwidth_insert_extract(
+; CHECK-KNL-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-KNL-NEXT: [[TMP1:%.*]] = load <8 x b64>, ptr [[PTR]], align 16
+; CHECK-KNL-NEXT: [[LA1:%.*]] = shufflevector <8 x b64> [[TMP1]], <8 x b64> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-KNL-NEXT: [[TMP2:%.*]] = bitcast <2 x b64> [[LA1]] to <2 x double>
+; CHECK-KNL-NEXT: [[LB2:%.*]] = shufflevector <8 x b64> [[TMP1]], <8 x b64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-KNL-NEXT: [[TMP3:%.*]] = bitcast <2 x b64> [[LB2]] to <2 x double>
+; CHECK-KNL-NEXT: [[LC3:%.*]] = shufflevector <8 x b64> [[TMP1]], <8 x b64> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-KNL-NEXT: [[TMP4:%.*]] = bitcast <2 x b64> [[LC3]] to <2 x double>
+; CHECK-KNL-NEXT: [[LD4:%.*]] = shufflevector <8 x b64> [[TMP1]], <8 x b64> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-KNL-NEXT: [[TMP5:%.*]] = bitcast <2 x b64> [[LD4]] to <2 x double>
+; CHECK-KNL-NEXT: [[V1:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-KNL-NEXT: [[V2:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
+; CHECK-KNL-NEXT: [[V3:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-KNL-NEXT: [[V4:%.*]] = extractelement <2 x double> [[TMP3]], i32 1
+; CHECK-KNL-NEXT: [[V5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-KNL-NEXT: [[V6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-KNL-NEXT: [[V7:%.*]] = extractelement <2 x double> [[TMP5]], i32 0
+; CHECK-KNL-NEXT: [[V8:%.*]] = extractelement <2 x double> [[TMP5]], i32 1
+; CHECK-KNL-NEXT: [[I1:%.*]] = insertelement <8 x double> poison, double [[V1]], i32 0
+; CHECK-KNL-NEXT: [[I2:%.*]] = insertelement <8 x double> [[I1]], double [[V2]], i32 1
+; CHECK-KNL-NEXT: [[I3:%.*]] = insertelement <8 x double> [[I2]], double [[V3]], i32 2
+; CHECK-KNL-NEXT: [[I4:%.*]] = insertelement <8 x double> [[I3]], double [[V4]], i32 3
+; CHECK-KNL-NEXT: [[I5:%.*]] = insertelement <8 x double> [[I4]], double [[V5]], i32 4
+; CHECK-KNL-NEXT: [[I6:%.*]] = insertelement <8 x double> [[I5]], double [[V6]], i32 5
+; CHECK-KNL-NEXT: [[I7:%.*]] = insertelement <8 x double> [[I6]], double [[V7]], i32 6
+; CHECK-KNL-NEXT: [[I8:%.*]] = insertelement <8 x double> [[I7]], double [[V8]], i32 7
+; CHECK-KNL-NEXT: ret <8 x double> [[I8]]
+;
+ %b = getelementptr <2 x double>, ptr %ptr, i32 1
+ %c = getelementptr <2 x double>, ptr %ptr, i32 2
+ %d = getelementptr <2 x double>, ptr %ptr, i32 3
+ %la = load <2 x double>, ptr %ptr
+ %lb = load <2 x double>, ptr %b
+ %lc = load <2 x double>, ptr %c
+ %ld = load <2 x double>, ptr %d
+ ; Scalarize everything - Explicitly not a shufflevector to test this code
+ ; path in the LSV
+ %v1 = extractelement <2 x double> %la, i32 0
+ %v2 = extractelement <2 x double> %la, i32 1
+ %v3 = extractelement <2 x double> %lb, i32 0
+ %v4 = extractelement <2 x double> %lb, i32 1
+ %v5 = extractelement <2 x double> %lc, i32 0
+ %v6 = extractelement <2 x double> %lc, i32 1
+ %v7 = extractelement <2 x double> %ld, i32 0
+ %v8 = extractelement <2 x double> %ld, i32 1
+ ; Make a vector again
+ %i1 = insertelement <8 x double> poison, double %v1, i32 0
+ %i2 = insertelement <8 x double> %i1, double %v2, i32 1
+ %i3 = insertelement <8 x double> %i2, double %v3, i32 2
+ %i4 = insertelement <8 x double> %i3, double %v4, i32 3
+ %i5 = insertelement <8 x double> %i4, double %v5, i32 4
+ %i6 = insertelement <8 x double> %i5, double %v6, i32 5
+ %i7 = insertelement <8 x double> %i6, double %v7, i32 6
+ %i8 = insertelement <8 x double> %i7, double %v8, i32 7
+ ret <8 x double> %i8
}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width.ll
index f225762d43801..3fb60f0aff3e0 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/load-width.ll
@@ -1,39 +1,95 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=load-store-vectorizer -mcpu haswell -S -o - %s | FileCheck --check-prefix=CHECK-HSW %s
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=load-store-vectorizer -mcpu knl -S -o - %s | FileCheck --check-prefix=CHECK-KNL %s
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -mcpu haswell -S -o - %s | FileCheck --check-prefix=CHECK-HSW %s
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -mcpu knl -S -o - %s | FileCheck --check-prefix=CHECK-KNL %s
define <8 x double> @loadwidth_insert_extract(ptr %ptr) {
- %b = getelementptr <2 x double>, ptr %ptr, i32 1
- %c = getelementptr <2 x double>, ptr %ptr, i32 2
- %d = getelementptr <2 x double>, ptr %ptr, i32 3
-; CHECK-HSW: load <4 x double>
-; CHECK-HSW: load <4 x double>
-; CHECK-HSW-NOT: load
-; CHECK-KNL: load <8 x double>
-; CHECK-KNL-NOT: load
- %la = load <2 x double>, ptr %ptr
- %lb = load <2 x double>, ptr %b
- %lc = load <2 x double>, ptr %c
- %ld = load <2 x double>, ptr %d
- ; Scalarize everything - Explicitly not a shufflevector to test this code
- ; path in the LSV
- %v1 = extractelement <2 x double> %la, i32 0
- %v2 = extractelement <2 x double> %la, i32 1
- %v3 = extractelement <2 x double> %lb, i32 0
- %v4 = extractelement <2 x double> %lb, i32 1
- %v5 = extractelement <2 x double> %lc, i32 0
- %v6 = extractelement <2 x double> %lc, i32 1
- %v7 = extractelement <2 x double> %ld, i32 0
- %v8 = extractelement <2 x double> %ld, i32 1
- ; Make a vector again
- %i1 = insertelement <8 x double> undef, double %v1, i32 0
- %i2 = insertelement <8 x double> %i1, double %v2, i32 1
- %i3 = insertelement <8 x double> %i2, double %v3, i32 2
- %i4 = insertelement <8 x double> %i3, double %v4, i32 3
- %i5 = insertelement <8 x double> %i4, double %v5, i32 4
- %i6 = insertelement <8 x double> %i5, double %v6, i32 5
- %i7 = insertelement <8 x double> %i6, double %v7, i32 6
- %i8 = insertelement <8 x double> %i7, double %v8, i32 7
- ret <8 x double> %i8
+; CHECK-HSW-LABEL: define <8 x double> @loadwidth_insert_extract(
+; CHECK-HSW-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-HSW-NEXT: [[C:%.*]] = getelementptr <2 x double>, ptr [[PTR]], i32 2
+; CHECK-HSW-NEXT: [[TMP1:%.*]] = load <4 x b64>, ptr [[PTR]], align 16
+; CHECK-HSW-NEXT: [[LA1:%.*]] = shufflevector <4 x b64> [[TMP1]], <4 x b64> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-HSW-NEXT: [[TMP2:%.*]] = bitcast <2 x b64> [[LA1]] to <2 x double>
+; CHECK-HSW-NEXT: [[LB2:%.*]] = shufflevector <4 x b64> [[TMP1]], <4 x b64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-HSW-NEXT: [[TMP3:%.*]] = bitcast <2 x b64> [[LB2]] to <2 x double>
+; CHECK-HSW-NEXT: [[TMP4:%.*]] = load <4 x b64>, ptr [[C]], align 16
+; CHECK-HSW-NEXT: [[LC3:%.*]] = shufflevector <4 x b64> [[TMP4]], <4 x b64> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-HSW-NEXT: [[TMP5:%.*]] = bitcast <2 x b64> [[LC3]] to <2 x double>
+; CHECK-HSW-NEXT: [[LD4:%.*]] = shufflevector <4 x b64> [[TMP4]], <4 x b64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-HSW-NEXT: [[TMP6:%.*]] = bitcast <2 x b64> [[LD4]] to <2 x double>
+; CHECK-HSW-NEXT: [[V1:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-HSW-NEXT: [[V2:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
+; CHECK-HSW-NEXT: [[V3:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-HSW-NEXT: [[V4:%.*]] = extractelement <2 x double> [[TMP3]], i32 1
+; CHECK-HSW-NEXT: [[V5:%.*]] = extractelement <2 x double> [[TMP5]], i32 0
+; CHECK-HSW-NEXT: [[V6:%.*]] = extractelement <2 x double> [[TMP5]], i32 1
+; CHECK-HSW-NEXT: [[V7:%.*]] = extractelement <2 x double> [[TMP6]], i32 0
+; CHECK-HSW-NEXT: [[V8:%.*]] = extractelement <2 x double> [[TMP6]], i32 1
+; CHECK-HSW-NEXT: [[I1:%.*]] = insertelement <8 x double> poison, double [[V1]], i32 0
+; CHECK-HSW-NEXT: [[I2:%.*]] = insertelement <8 x double> [[I1]], double [[V2]], i32 1
+; CHECK-HSW-NEXT: [[I3:%.*]] = insertelement <8 x double> [[I2]], double [[V3]], i32 2
+; CHECK-HSW-NEXT: [[I4:%.*]] = insertelement <8 x double> [[I3]], double [[V4]], i32 3
+; CHECK-HSW-NEXT: [[I5:%.*]] = insertelement <8 x double> [[I4]], double [[V5]], i32 4
+; CHECK-HSW-NEXT: [[I6:%.*]] = insertelement <8 x double> [[I5]], double [[V6]], i32 5
+; CHECK-HSW-NEXT: [[I7:%.*]] = insertelement <8 x double> [[I6]], double [[V7]], i32 6
+; CHECK-HSW-NEXT: [[I8:%.*]] = insertelement <8 x double> [[I7]], double [[V8]], i32 7
+; CHECK-HSW-NEXT: ret <8 x double> [[I8]]
+;
+; CHECK-KNL-LABEL: define <8 x double> @loadwidth_insert_extract(
+; CHECK-KNL-SAME: ptr [[PTR:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-KNL-NEXT: [[TMP1:%.*]] = load <8 x b64>, ptr [[PTR]], align 16
+; CHECK-KNL-NEXT: [[LA1:%.*]] = shufflevector <8 x b64> [[TMP1]], <8 x b64> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-KNL-NEXT: [[TMP2:%.*]] = bitcast <2 x b64> [[LA1]] to <2 x double>
+; CHECK-KNL-NEXT: [[LB2:%.*]] = shufflevector <8 x b64> [[TMP1]], <8 x b64> poison, <2 x i32> <i32 2, i32 3>
+; CHECK-KNL-NEXT: [[TMP3:%.*]] = bitcast <2 x b64> [[LB2]] to <2 x double>
+; CHECK-KNL-NEXT: [[LC3:%.*]] = shufflevector <8 x b64> [[TMP1]], <8 x b64> poison, <2 x i32> <i32 4, i32 5>
+; CHECK-KNL-NEXT: [[TMP4:%.*]] = bitcast <2 x b64> [[LC3]] to <2 x double>
+; CHECK-KNL-NEXT: [[LD4:%.*]] = shufflevector <8 x b64> [[TMP1]], <8 x b64> poison, <2 x i32> <i32 6, i32 7>
+; CHECK-KNL-NEXT: [[TMP5:%.*]] = bitcast <2 x b64> [[LD4]] to <2 x double>
+; CHECK-KNL-NEXT: [[V1:%.*]] = extractelement <2 x double> [[TMP2]], i32 0
+; CHECK-KNL-NEXT: [[V2:%.*]] = extractelement <2 x double> [[TMP2]], i32 1
+; CHECK-KNL-NEXT: [[V3:%.*]] = extractelement <2 x double> [[TMP3]], i32 0
+; CHECK-KNL-NEXT: [[V4:%.*]] = extractelement <2 x double> [[TMP3]], i32 1
+; CHECK-KNL-NEXT: [[V5:%.*]] = extractelement <2 x double> [[TMP4]], i32 0
+; CHECK-KNL-NEXT: [[V6:%.*]] = extractelement <2 x double> [[TMP4]], i32 1
+; CHECK-KNL-NEXT: [[V7:%.*]] = extractelement <2 x double> [[TMP5]], i32 0
+; CHECK-KNL-NEXT: [[V8:%.*]] = extractelement <2 x double> [[TMP5]], i32 1
+; CHECK-KNL-NEXT: [[I1:%.*]] = insertelement <8 x double> poison, double [[V1]], i32 0
+; CHECK-KNL-NEXT: [[I2:%.*]] = insertelement <8 x double> [[I1]], double [[V2]], i32 1
+; CHECK-KNL-NEXT: [[I3:%.*]] = insertelement <8 x double> [[I2]], double [[V3]], i32 2
+; CHECK-KNL-NEXT: [[I4:%.*]] = insertelement <8 x double> [[I3]], double [[V4]], i32 3
+; CHECK-KNL-NEXT: [[I5:%.*]] = insertelement <8 x double> [[I4]], double [[V5]], i32 4
+; CHECK-KNL-NEXT: [[I6:%.*]] = insertelement <8 x double> [[I5]], double [[V6]], i32 5
+; CHECK-KNL-NEXT: [[I7:%.*]] = insertelement <8 x double> [[I6]], double [[V7]], i32 6
+; CHECK-KNL-NEXT: [[I8:%.*]] = insertelement <8 x double> [[I7]], double [[V8]], i32 7
+; CHECK-KNL-NEXT: ret <8 x double> [[I8]]
+;
+ %b = getelementptr <2 x double>, ptr %ptr, i32 1
+ %c = getelementptr <2 x double>, ptr %ptr, i32 2
+ %d = getelementptr <2 x double>, ptr %ptr, i32 3
+ %la = load <2 x double>, ptr %ptr
+ %lb = load <2 x double>, ptr %b
+ %lc = load <2 x double>, ptr %c
+ %ld = load <2 x double>, ptr %d
+ ; Scalarize everything - Explicitly not a shufflevector to test this code
+ ; path in the LSV
+ %v1 = extractelement <2 x double> %la, i32 0
+ %v2 = extractelement <2 x double> %la, i32 1
+ %v3 = extractelement <2 x double> %lb, i32 0
+ %v4 = extractelement <2 x double> %lb, i32 1
+ %v5 = extractelement <2 x double> %lc, i32 0
+ %v6 = extractelement <2 x double> %lc, i32 1
+ %v7 = extractelement <2 x double> %ld, i32 0
+ %v8 = extractelement <2 x double> %ld, i32 1
+ ; Make a vector again
+ %i1 = insertelement <8 x double> poison, double %v1, i32 0
+ %i2 = insertelement <8 x double> %i1, double %v2, i32 1
+ %i3 = insertelement <8 x double> %i2, double %v3, i32 2
+ %i4 = insertelement <8 x double> %i3, double %v4, i32 3
+ %i5 = insertelement <8 x double> %i4, double %v5, i32 4
+ %i6 = insertelement <8 x double> %i5, double %v6, i32 5
+ %i7 = insertelement <8 x double> %i6, double %v7, i32 6
+ %i8 = insertelement <8 x double> %i7, double %v8, i32 7
+ ret <8 x double> %i8
}
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
index 6a7a5dac40c83..e85d8f81bb4d5 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/massive_indirection.ll
@@ -19,7 +19,7 @@ define void @v1_v2_v4_v1_to_v8_levels_6_7_8_8(i32 %arg0, ptr align 16 %arg1) {
; CHECK-NEXT: [[LEVEL4:%.*]] = getelementptr i8, ptr [[LEVEL3]], i32 [[ARG0]]
; CHECK-NEXT: [[LEVEL5:%.*]] = getelementptr i8, ptr [[LEVEL4]], i32 [[ARG0]]
; CHECK-NEXT: [[A6:%.*]] = getelementptr i8, ptr [[LEVEL5]], i32 [[ARG0]]
-; CHECK-NEXT: store <8 x half> zeroinitializer, ptr [[A6]], align 16
+; CHECK-NEXT: store <8 x b16> zeroinitializer, ptr [[A6]], align 16
; CHECK-NEXT: ret void
;
@@ -50,7 +50,7 @@ define void @v1x8_levels_6_7_8_9_10_11_12_13(i32 %arg0, ptr align 16 %arg1) {
; CHECK-NEXT: [[LEVEL4:%.*]] = getelementptr i8, ptr [[LEVEL3]], i32 [[ARG0]]
; CHECK-NEXT: [[LEVEL5:%.*]] = getelementptr i8, ptr [[LEVEL4]], i32 [[ARG0]]
; CHECK-NEXT: [[A6:%.*]] = getelementptr i8, ptr [[LEVEL5]], i32 [[ARG0]]
-; CHECK-NEXT: store <8 x half> zeroinitializer, ptr [[A6]], align 16
+; CHECK-NEXT: store <8 x b16> zeroinitializer, ptr [[A6]], align 16
; CHECK-NEXT: ret void
;
@@ -95,17 +95,20 @@ define void @v1_4_4_4_2_1_to_v8_8_levels_6_7(i32 %arg0, ptr addrspace(3) align 1
; CHECK: [[_LR_PH:.*:]]
; CHECK-NEXT: [[GEP:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[TMP5]], i32 [[ARG4]]
; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr addrspace(3) [[GEP]], i32 [[ARG5]]
-; CHECK-NEXT: [[TMP7:%.*]] = insertelement <8 x half> poison, half [[ARG6_HALF]], i32 0
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <8 x half> [[TMP7]], half 0.000000e+00, i32 1
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x half> [[TMP8]], half 0.000000e+00, i32 2
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <8 x half> [[TMP9]], half 0.000000e+00, i32 3
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x half> [[TMP10]], half 0.000000e+00, i32 4
-; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x half> [[ARG8_2XHALF]], i32 0
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x half> [[TMP11]], half [[TMP12]], i32 5
-; CHECK-NEXT: [[TMP14:%.*]] = extractelement <2 x half> [[ARG8_2XHALF]], i32 1
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x half> [[TMP13]], half [[TMP14]], i32 6
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x half> [[TMP15]], half [[ARG7_HALF]], i32 7
-; CHECK-NEXT: store <8 x half> [[TMP16]], ptr addrspace(3) [[TMP6]], align 2
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast half [[ARG6_HALF]] to b16
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <8 x b16> poison, b16 [[TMP14]], i32 0
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <8 x b16> [[TMP15]], b16 0, i32 1
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <8 x b16> [[TMP16]], b16 0, i32 2
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <8 x b16> [[TMP17]], b16 0, i32 3
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <8 x b16> [[TMP18]], b16 0, i32 4
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast <2 x half> [[ARG8_2XHALF]] to <2 x b16>
+; CHECK-NEXT: [[TMP8:%.*]] = extractelement <2 x b16> [[TMP7]], i32 0
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <8 x b16> [[TMP19]], b16 [[TMP8]], i32 5
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x b16> [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <8 x b16> [[TMP9]], b16 [[TMP10]], i32 6
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast half [[ARG7_HALF]] to b16
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <8 x b16> [[TMP11]], b16 [[TMP12]], i32 7
+; CHECK-NEXT: store <8 x b16> [[TMP13]], ptr addrspace(3) [[TMP6]], align 2
; CHECK-NEXT: br [[DOTEXIT_POINT]]
; CHECK: [[_EXIT_POINT:.*:]]
; CHECK-NEXT: ret void
@@ -155,9 +158,11 @@ define void @variadics1(ptr %vlist) {
; CHECK-NEXT: [[ARGP_NEXT12:%.*]] = getelementptr i8, ptr [[ARGP_CUR11_ALIGNED]], i64 8
; CHECK-NEXT: [[X2:%.*]] = getelementptr i8, ptr [[ARGP_NEXT12]], i32 7
; CHECK-NEXT: [[ARGP_CUR16_ALIGNED:%.*]] = call ptr @llvm.ptrmask.p0.i64(ptr [[X2]], i64 0)
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[ARGP_CUR16_ALIGNED]], align 4294967296
-; CHECK-NEXT: [[X31:%.*]] = extractelement <2 x double> [[TMP1]], i32 0
-; CHECK-NEXT: [[X42:%.*]] = extractelement <2 x double> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b64>, ptr [[ARGP_CUR16_ALIGNED]], align 4294967296
+; CHECK-NEXT: [[X32:%.*]] = extractelement <2 x b64> [[TMP1]], i32 0
+; CHECK-NEXT: [[X31:%.*]] = bitcast b64 [[X32]] to double
+; CHECK-NEXT: [[X43:%.*]] = extractelement <2 x b64> [[TMP1]], i32 1
+; CHECK-NEXT: [[X42:%.*]] = bitcast b64 [[X43]] to double
; CHECK-NEXT: [[X5:%.*]] = fadd double [[X42]], [[X31]]
; CHECK-NEXT: store double [[X5]], ptr null, align 8
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/merge-tbaa.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/merge-tbaa.ll
index 20e1fa8633b07..3d4020c0ea68c 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/merge-tbaa.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/merge-tbaa.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=load-store-vectorizer -S < %s | \
; RUN: FileCheck %s
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S < %s | \
@@ -21,10 +22,24 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
; return p->f;
; }
define float @foo(ptr %p) {
+; CHECK-LABEL: define float @foo(
+; CHECK-SAME: ptr [[P:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr [[P]], align 4, !tbaa [[CHAR_TBAA0:![0-9]+]]
+; CHECK-NEXT: [[TMP1:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[TMP1]] to float
+; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[TMP3]] to i32
+; CHECK-NEXT: [[SUB:%.*]] = fadd float [[TMP2]], -1.000000e+00
+; CHECK-NEXT: [[SUB1:%.*]] = add nsw i32 [[TMP4]], -1
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast float [[SUB]] to b32
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x b32> poison, b32 [[TMP5]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i32 [[SUB1]] to b32
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <2 x b32> [[TMP6]], b32 [[TMP7]], i32 1
+; CHECK-NEXT: store <2 x b32> [[TMP8]], ptr [[P]], align 4, !tbaa [[CHAR_TBAA0]]
+; CHECK-NEXT: ret float [[SUB]]
+;
entry:
-; CHECK-LABEL: foo
-; CHECK: load <2 x i32>, {{.*}}, !tbaa [[TAG_char:!.*]]
-; CHECK: store <2 x i32> {{.*}}, !tbaa [[TAG_char]]
%0 = load float, ptr %p, align 4, !tbaa !2
%sub = fadd float %0, -1.000000e+00
store float %sub, ptr %p, align 4, !tbaa !2
@@ -43,5 +58,8 @@ entry:
!7 = !{!"int", !5, i64 0}
!8 = !{!3, !7, i64 4}
-; CHECK-DAG: [[TYPE_char:!.*]] = !{!"omnipotent char", {{.*}}, i64 0}
-; CHECK-DAG: [[TAG_char]] = !{[[TYPE_char]], [[TYPE_char]], i64 0}
+;.
+; CHECK: [[CHAR_TBAA0]] = !{[[META1:![0-9]+]], [[META1]], i64 0}
+; CHECK: [[META1]] = !{!"omnipotent char", [[META2:![0-9]+]], i64 0}
+; CHECK: [[META2]] = !{!"Simple C++ TBAA"}
+;.
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/opaque-ptr.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/opaque-ptr.ll
index 12d06df448fbb..9fb0e5de7d5cf 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/opaque-ptr.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/opaque-ptr.ll
@@ -3,10 +3,12 @@
define void @test(ptr %ptr) {
; CHECK-LABEL: @test(
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[PTR:%.*]], align 4
-; CHECK-NEXT: [[L11:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[L22:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: store <2 x i32> zeroinitializer, ptr [[PTR]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[PTR:%.*]], align 4
+; CHECK-NEXT: [[L11:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[L11]] to i32
+; CHECK-NEXT: [[L22:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[L22]] to i32
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr [[PTR]], align 4
; CHECK-NEXT: ret void
;
%ptr2 = getelementptr i32, ptr %ptr, i64 1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/or-disjoint-nested-add.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/or-disjoint-nested-add.ll
index ed77318936ff3..4039d79820164 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/or-disjoint-nested-add.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/or-disjoint-nested-add.ll
@@ -23,11 +23,15 @@ define void @or_disjoint_inner(i32 %v0, i32 %v1, ptr %src, ptr %dst) {
; CHECK-NEXT: [[A:%.*]] = add nsw i32 [[V1]], [[V0]]
; CHECK-NEXT: [[EA:%.*]] = sext i32 [[A]] to i64
; CHECK-NEXT: [[GA:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[EA]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[GA]], align 1
-; CHECK-NEXT: [[LA1:%.*]] = extractelement <4 x i8> [[TMP0]], i32 0
-; CHECK-NEXT: [[LB2:%.*]] = extractelement <4 x i8> [[TMP0]], i32 1
-; CHECK-NEXT: [[LC3:%.*]] = extractelement <4 x i8> [[TMP0]], i32 2
-; CHECK-NEXT: [[LD4:%.*]] = extractelement <4 x i8> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b8>, ptr [[GA]], align 1
+; CHECK-NEXT: [[LA2:%.*]] = extractelement <4 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[LA1:%.*]] = bitcast b8 [[LA2]] to i8
+; CHECK-NEXT: [[LB3:%.*]] = extractelement <4 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[LB2:%.*]] = bitcast b8 [[LB3]] to i8
+; CHECK-NEXT: [[LC4:%.*]] = extractelement <4 x b8> [[TMP0]], i32 2
+; CHECK-NEXT: [[LC3:%.*]] = bitcast b8 [[LC4]] to i8
+; CHECK-NEXT: [[LD5:%.*]] = extractelement <4 x b8> [[TMP0]], i32 3
+; CHECK-NEXT: [[LD4:%.*]] = bitcast b8 [[LD5]] to i8
; CHECK-NEXT: [[R0:%.*]] = insertelement <4 x i8> poison, i8 [[LA1]], i32 0
; CHECK-NEXT: [[R1:%.*]] = insertelement <4 x i8> [[R0]], i8 [[LB2]], i32 1
; CHECK-NEXT: [[R2:%.*]] = insertelement <4 x i8> [[R1]], i8 [[LC3]], i32 2
@@ -78,11 +82,15 @@ define void @or_disjoint_both_inner(i32 %v0, i32 %v1, ptr %src, ptr %dst) {
; CHECK-NEXT: [[A:%.*]] = add nsw i32 [[V1]], [[INNER_A]]
; CHECK-NEXT: [[EA:%.*]] = sext i32 [[A]] to i64
; CHECK-NEXT: [[GA:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[EA]]
-; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr [[GA]], align 1
-; CHECK-NEXT: [[LA1:%.*]] = extractelement <4 x i8> [[TMP0]], i32 0
-; CHECK-NEXT: [[LB2:%.*]] = extractelement <4 x i8> [[TMP0]], i32 1
-; CHECK-NEXT: [[LC3:%.*]] = extractelement <4 x i8> [[TMP0]], i32 2
-; CHECK-NEXT: [[LD4:%.*]] = extractelement <4 x i8> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b8>, ptr [[GA]], align 1
+; CHECK-NEXT: [[LA2:%.*]] = extractelement <4 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[LA1:%.*]] = bitcast b8 [[LA2]] to i8
+; CHECK-NEXT: [[LB3:%.*]] = extractelement <4 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[LB2:%.*]] = bitcast b8 [[LB3]] to i8
+; CHECK-NEXT: [[LC4:%.*]] = extractelement <4 x b8> [[TMP0]], i32 2
+; CHECK-NEXT: [[LC3:%.*]] = bitcast b8 [[LC4]] to i8
+; CHECK-NEXT: [[LD5:%.*]] = extractelement <4 x b8> [[TMP0]], i32 3
+; CHECK-NEXT: [[LD4:%.*]] = bitcast b8 [[LD5]] to i8
; CHECK-NEXT: [[R0:%.*]] = insertelement <4 x i8> poison, i8 [[LA1]], i32 0
; CHECK-NEXT: [[R1:%.*]] = insertelement <4 x i8> [[R0]], i8 [[LB2]], i32 1
; CHECK-NEXT: [[R2:%.*]] = insertelement <4 x i8> [[R1]], i8 [[LC3]], i32 2
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/or-disjoint-zext.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/or-disjoint-zext.ll
index 4ec22a18479b7..6846558171bb2 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/or-disjoint-zext.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/or-disjoint-zext.ll
@@ -12,10 +12,13 @@ define void @test_zext_add(ptr %base, i32 %idx) {
; CHECK-SAME: ptr [[BASE:%.*]], i32 [[IDX:%.*]]) {
; CHECK-NEXT: [[I0:%.*]] = zext i32 [[IDX]] to i64
; CHECK-NEXT: [[G0:%.*]] = getelementptr inbounds i32, ptr [[BASE]], i64 [[I0]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[G0]], align 4
-; CHECK-NEXT: [[V01:%.*]] = extractelement <3 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[V12:%.*]] = extractelement <3 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[V23:%.*]] = extractelement <3 x i32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP1:%.*]] = load <3 x b32>, ptr [[G0]], align 4
+; CHECK-NEXT: [[V1:%.*]] = extractelement <3 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[V01:%.*]] = bitcast b32 [[V1]] to i32
+; CHECK-NEXT: [[V13:%.*]] = extractelement <3 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[V12:%.*]] = bitcast b32 [[V13]] to i32
+; CHECK-NEXT: [[V24:%.*]] = extractelement <3 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[V23:%.*]] = bitcast b32 [[V24]] to i32
; CHECK-NEXT: [[IDX3:%.*]] = add nuw i32 [[IDX]], 3
; CHECK-NEXT: [[I3:%.*]] = zext i32 [[IDX3]] to i64
; CHECK-NEXT: [[G3:%.*]] = getelementptr inbounds i32, ptr [[BASE]], i64 [[I3]]
@@ -62,10 +65,13 @@ define void @test_zext_or_disjoint(ptr %base, i32 %idx) {
; CHECK-SAME: ptr [[BASE:%.*]], i32 [[IDX:%.*]]) {
; CHECK-NEXT: [[I0:%.*]] = zext i32 [[IDX]] to i64
; CHECK-NEXT: [[G0:%.*]] = getelementptr inbounds i32, ptr [[BASE]], i64 [[I0]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr [[G0]], align 4
-; CHECK-NEXT: [[V01:%.*]] = extractelement <3 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[V12:%.*]] = extractelement <3 x i32> [[TMP1]], i32 1
-; CHECK-NEXT: [[V23:%.*]] = extractelement <3 x i32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP1:%.*]] = load <3 x b32>, ptr [[G0]], align 4
+; CHECK-NEXT: [[V1:%.*]] = extractelement <3 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[V01:%.*]] = bitcast b32 [[V1]] to i32
+; CHECK-NEXT: [[V13:%.*]] = extractelement <3 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[V12:%.*]] = bitcast b32 [[V13]] to i32
+; CHECK-NEXT: [[V24:%.*]] = extractelement <3 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[V23:%.*]] = bitcast b32 [[V24]] to i32
; CHECK-NEXT: [[B3:%.*]] = or disjoint i32 [[IDX]], 3
; CHECK-NEXT: [[I3:%.*]] = zext i32 [[B3]] to i64
; CHECK-NEXT: [[G3:%.*]] = getelementptr inbounds i32, ptr [[BASE]], i64 [[I3]]
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order32.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order32.ll
index 897c39a8cbb5a..5ca5e86aeed73 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order32.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order32.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=x86_64-unknown-linux -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; RUN: opt -mtriple=x86_64-unknown-linux -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
@@ -11,11 +12,19 @@ target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:
; and instructions which were between the first and last load should be
; reordered preserving their relative order inasmuch as possible.
-; CHECK-LABEL: @preserve_order_32(
-; CHECK: load <2 x i32>
-; CHECK: %buff.val = load i8
-; CHECK: store i8 0
define void @preserve_order_32(ptr noalias %buff) #0 {
+; CHECK-LABEL: define void @preserve_order_32(
+; CHECK-SAME: ptr noalias [[BUFF:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b32>, ptr [[BUFF]], align 8
+; CHECK-NEXT: [[BUFF_INT1:%.*]] = extractelement <2 x b32> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b32 [[BUFF_INT1]] to i32
+; CHECK-NEXT: [[BUFF_P2:%.*]] = extractelement <2 x b32> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[BUFF_P2]] to ptr
+; CHECK-NEXT: [[BUFF_VAL:%.*]] = load i8, ptr [[TMP2]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP2]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%tmp1 = getelementptr inbounds %struct.buffer_t, ptr %buff, i32 0, i32 1
%buff.p = load ptr, ptr %tmp1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order64.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order64.ll
index d5b91fea5da36..de733036b9925 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order64.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/preserve-order64.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
@@ -12,11 +13,19 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
; and instructions which were between the first and last load should be
; reordered preserving their relative order inasmuch as possible.
-; CHECK-LABEL: @preserve_order_64(
-; CHECK: load <2 x i64>
-; CHECK: %buff.val = load i8
-; CHECK: store i8 0
define void @preserve_order_64(ptr noalias %buff) #0 {
+; CHECK-LABEL: define void @preserve_order_64(
+; CHECK-SAME: ptr noalias [[BUFF:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b64>, ptr [[BUFF]], align 16
+; CHECK-NEXT: [[BUFF_INT1:%.*]] = extractelement <2 x b64> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b64 [[BUFF_INT1]] to i64
+; CHECK-NEXT: [[BUFF_P2:%.*]] = extractelement <2 x b64> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[BUFF_P2]] to ptr
+; CHECK-NEXT: [[BUFF_VAL:%.*]] = load i8, ptr [[TMP2]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP2]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%tmp1 = getelementptr inbounds %struct.buffer_t, ptr %buff, i64 0, i32 1
%buff.p = load ptr, ptr %tmp1
@@ -28,11 +37,19 @@ entry:
; Check reordering recurses correctly.
-; CHECK-LABEL: @transitive_reorder(
-; CHECK: load <2 x i64>
-; CHECK: %buff.val = load i8
-; CHECK: store i8 0
define void @transitive_reorder(ptr noalias %buff, ptr noalias %nest) #0 {
+; CHECK-LABEL: define void @transitive_reorder(
+; CHECK-SAME: ptr noalias [[BUFF:%.*]], ptr noalias [[NEST:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b64>, ptr [[NEST]], align 16
+; CHECK-NEXT: [[BUFF_INT1:%.*]] = extractelement <2 x b64> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b64 [[BUFF_INT1]] to i64
+; CHECK-NEXT: [[BUFF_P2:%.*]] = extractelement <2 x b64> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b64 [[BUFF_P2]] to ptr
+; CHECK-NEXT: [[BUFF_VAL:%.*]] = load i8, ptr [[TMP2]], align 1
+; CHECK-NEXT: store i8 0, ptr [[TMP2]], align 8
+; CHECK-NEXT: ret void
+;
entry:
%tmp1 = getelementptr inbounds %struct.buffer_t, ptr %nest, i64 0, i32 1
%buff.p = load ptr, ptr %tmp1
@@ -44,12 +61,24 @@ entry:
; Check for no vectorization over phi node
-; CHECK-LABEL: @no_vect_phi(
-; CHECK: load ptr
-; CHECK: load i8
-; CHECK: store i8 0
-; CHECK: load i64
define void @no_vect_phi(ptr noalias %ptr, ptr noalias %buff) {
+; CHECK-LABEL: define void @no_vect_phi(
+; CHECK-SAME: ptr noalias [[PTR:%.*]], ptr noalias [[BUFF:%.*]]) {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr inbounds [[STRUCT_BUFFER_T:%.*]], ptr [[BUFF]], i64 0, i32 1
+; CHECK-NEXT: [[BUFF_P:%.*]] = load ptr, ptr [[TMP1]], align 8
+; CHECK-NEXT: [[BUFF_VAL:%.*]] = load i8, ptr [[BUFF_P]], align 1
+; CHECK-NEXT: store i8 0, ptr [[BUFF_P]], align 8
+; CHECK-NEXT: br label %"for something"
+; CHECK: "for something":
+; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %"for something" ]
+; CHECK-NEXT: [[BUFF_INT:%.*]] = load i64, ptr [[BUFF]], align 16
+; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; CHECK-NEXT: [[CMP_RES:%.*]] = icmp eq i64 [[INDEX_NEXT]], 8
+; CHECK-NEXT: br i1 [[CMP_RES]], label %[[ENDING:.*]], label %"for something"
+; CHECK: [[ENDING]]:
+; CHECK-NEXT: ret void
+;
entry:
%tmp1 = getelementptr inbounds %struct.buffer_t, ptr %buff, i64 0, i32 1
%buff.p = load ptr, ptr %tmp1
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/scev-range-zext.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/scev-range-zext.ll
index 74fe2bc4f1c1a..b3df5ce2441f1 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/scev-range-zext.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/scev-range-zext.ll
@@ -25,9 +25,11 @@ define void @scev_range_urem_bounded(ptr noalias %in, ptr noalias %out, i32 %idx
; CHECK-NEXT: [[BOUNDED:%.*]] = urem i32 [[IDX]], -4
; CHECK-NEXT: [[EXT0:%.*]] = zext i32 [[BOUNDED]] to i64
; CHECK-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr [[IN]], i64 [[EXT0]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[GEP0]], align 4
-; CHECK-NEXT: [[V01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[V42:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[GEP0]], align 4
+; CHECK-NEXT: [[V1:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[V01:%.*]] = bitcast b32 [[V1]] to i32
+; CHECK-NEXT: [[V43:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[V42:%.*]] = bitcast b32 [[V43]] to i32
; CHECK-NEXT: [[S:%.*]] = add i32 [[V01]], [[V42]]
; CHECK-NEXT: store i32 [[S]], ptr [[OUT]], align 4
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/subchain-interleaved.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/subchain-interleaved.ll
index 8f9a86e016702..a83176800f0a7 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/subchain-interleaved.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/subchain-interleaved.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=load-store-vectorizer -S -o - %s | FileCheck %s
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -S -o - %s | FileCheck %s
@@ -6,11 +7,23 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128"
; Vectorized subsets of the load/store chains in the presence of
; interleaved loads/stores
-; CHECK-LABEL: @interleave_2L_2S(
-; CHECK: load <2 x i32>
-; CHECK: store <2 x i32>
-; CHECK: load <2 x i32>
define void @interleave_2L_2S(ptr noalias %ptr) {
+; CHECK-LABEL: define void @interleave_2L_2S(
+; CHECK-SAME: ptr noalias [[PTR:%.*]]) {
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i32, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[PTR]], align 4
+; CHECK-NEXT: [[L21:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[L21]] to i32
+; CHECK-NEXT: [[L12:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[L12]] to i32
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr [[PTR]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x b32>, ptr [[NEXT_GEP1]], align 4
+; CHECK-NEXT: [[L33:%.*]] = extractelement <2 x b32> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[L33]] to i32
+; CHECK-NEXT: [[L44:%.*]] = extractelement <2 x b32> [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[L44]] to i32
+; CHECK-NEXT: ret void
+;
%next.gep1 = getelementptr i32, ptr %ptr, i64 1
%next.gep2 = getelementptr i32, ptr %ptr, i64 2
@@ -24,12 +37,24 @@ define void @interleave_2L_2S(ptr noalias %ptr) {
ret void
}
-; CHECK-LABEL: @interleave_3L_2S_1L(
-; CHECK: load <2 x i32>
-; CHECK: store <2 x i32>
-; CHECK: load <2 x i32>
define void @interleave_3L_2S_1L(ptr noalias %ptr) {
+; CHECK-LABEL: define void @interleave_3L_2S_1L(
+; CHECK-SAME: ptr noalias [[PTR:%.*]]) {
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i32, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[PTR]], align 4
+; CHECK-NEXT: [[L21:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[L21]] to i32
+; CHECK-NEXT: [[L12:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[L12]] to i32
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr [[PTR]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x b32>, ptr [[NEXT_GEP1]], align 4
+; CHECK-NEXT: [[L33:%.*]] = extractelement <2 x b32> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[L33]] to i32
+; CHECK-NEXT: [[L44:%.*]] = extractelement <2 x b32> [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[L44]] to i32
+; CHECK-NEXT: ret void
+;
%next.gep1 = getelementptr i32, ptr %ptr, i64 1
%next.gep2 = getelementptr i32, ptr %ptr, i64 2
@@ -43,11 +68,19 @@ define void @interleave_3L_2S_1L(ptr noalias %ptr) {
ret void
}
-; CHECK-LABEL: @chain_suffix(
-; CHECK: load i32
-; CHECK: store <2 x i32>
-; CHECK: load <2 x i32>
define void @chain_suffix(ptr noalias %ptr) {
+; CHECK-LABEL: define void @chain_suffix(
+; CHECK-SAME: ptr noalias [[PTR:%.*]]) {
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i32, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[L2:%.*]] = load i32, ptr [[PTR]], align 4
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr [[PTR]], align 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[NEXT_GEP1]], align 4
+; CHECK-NEXT: [[L31:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[L31]] to i32
+; CHECK-NEXT: [[L42:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[L42]] to i32
+; CHECK-NEXT: ret void
+;
%next.gep1 = getelementptr i32, ptr %ptr, i64 1
%next.gep2 = getelementptr i32, ptr %ptr, i64 2
@@ -61,11 +94,25 @@ define void @chain_suffix(ptr noalias %ptr) {
}
-; CHECK-LABEL: @chain_prefix_suffix(
-; CHECK: load <2 x i32>
-; CHECK: store <2 x i32>
-; CHECK: load <3 x i32>
define void @chain_prefix_suffix(ptr noalias %ptr) {
+; CHECK-LABEL: define void @chain_prefix_suffix(
+; CHECK-SAME: ptr noalias [[PTR:%.*]]) {
+; CHECK-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i32, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[PTR]], align 4
+; CHECK-NEXT: [[L11:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[L11]] to i32
+; CHECK-NEXT: [[L22:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[L22]] to i32
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr [[NEXT_GEP1]], align 4
+; CHECK-NEXT: [[TMP4:%.*]] = load <3 x b32>, ptr [[NEXT_GEP1]], align 4
+; CHECK-NEXT: [[L33:%.*]] = extractelement <3 x b32> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[L33]] to i32
+; CHECK-NEXT: [[L44:%.*]] = extractelement <3 x b32> [[TMP4]], i32 1
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[L44]] to i32
+; CHECK-NEXT: [[L55:%.*]] = extractelement <3 x b32> [[TMP4]], i32 2
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast b32 [[L55]] to i32
+; CHECK-NEXT: ret void
+;
%next.gep1 = getelementptr i32, ptr %ptr, i64 1
%next.gep2 = getelementptr i32, ptr %ptr, i64 2
%next.gep3 = getelementptr i32, ptr %ptr, i64 3
@@ -81,13 +128,30 @@ define void @chain_prefix_suffix(ptr noalias %ptr) {
ret void
}
-; CHECK-LABEL: @interleave_get_longest
-; CHECK: load <2 x i32>
-; CHECK: store <2 x i32> zeroinitializer
-; CHECK: load <3 x i32>
-; CHECK: load i32
define void @interleave_get_longest(ptr noalias %ptr) {
+; CHECK-LABEL: define void @interleave_get_longest(
+; CHECK-SAME: ptr noalias [[PTR:%.*]]) {
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i32, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[GEP4:%.*]] = getelementptr i32, ptr [[PTR]], i64 4
+; CHECK-NEXT: [[V2:%.*]] = load <2 x b32>, ptr [[PTR]], align 4
+; CHECK-NEXT: [[L21:%.*]] = extractelement <2 x b32> [[V2]], i32 0
+; CHECK-NEXT: [[BC0:%.*]] = bitcast b32 [[L21]] to i32
+; CHECK-NEXT: [[L12:%.*]] = extractelement <2 x b32> [[V2]], i32 1
+; CHECK-NEXT: [[BC1:%.*]] = bitcast b32 [[L12]] to i32
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr [[PTR]], align 4
+; CHECK-NEXT: [[V3:%.*]] = load <3 x b32>, ptr [[GEP1]], align 4
+; CHECK-NEXT: [[L33:%.*]] = extractelement <3 x b32> [[V3]], i32 0
+; CHECK-NEXT: [[BC2:%.*]] = bitcast b32 [[L33]] to i32
+; CHECK-NEXT: [[L44:%.*]] = extractelement <3 x b32> [[V3]], i32 1
+; CHECK-NEXT: [[BC3:%.*]] = bitcast b32 [[L44]] to i32
+; CHECK-NEXT: [[L55:%.*]] = extractelement <3 x b32> [[V3]], i32 2
+; CHECK-NEXT: [[BC4:%.*]] = bitcast b32 [[L55]] to i32
+; CHECK-NEXT: [[V4:%.*]] = load b32, ptr [[GEP4]], align 4
+; CHECK-NEXT: [[BC5:%.*]] = bitcast b32 [[V4]] to i32
+; CHECK-NEXT: [[BC6:%.*]] = bitcast b32 [[V4]] to i32
+; CHECK-NEXT: ret void
+;
%tmp2 = getelementptr i32, ptr %ptr, i64 1
%tmp3 = getelementptr i32, ptr %ptr, i64 2
%tmp4 = getelementptr i32, ptr %ptr, i64 3
@@ -106,12 +170,30 @@ define void @interleave_get_longest(ptr noalias %ptr) {
ret void
}
-; CHECK-LABEL: @interleave_get_longest_aligned
-; CHECK: load <2 x i32>
-; CHECK: store <2 x i32> zeroinitializer
-; CHECK: load <4 x i32>
define void @interleave_get_longest_aligned(ptr noalias %ptr) {
+; CHECK-LABEL: define void @interleave_get_longest_aligned(
+; CHECK-SAME: ptr noalias [[PTR:%.*]]) {
+; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i32, ptr [[PTR]], i64 1
+; CHECK-NEXT: [[V1:%.*]] = load <2 x b32>, ptr [[PTR]], align 4
+; CHECK-NEXT: [[L21:%.*]] = extractelement <2 x b32> [[V1]], i32 0
+; CHECK-NEXT: [[BC0:%.*]] = bitcast b32 [[L21]] to i32
+; CHECK-NEXT: [[L12:%.*]] = extractelement <2 x b32> [[V1]], i32 1
+; CHECK-NEXT: [[BC1:%.*]] = bitcast b32 [[L12]] to i32
+; CHECK-NEXT: store <2 x b32> zeroinitializer, ptr [[PTR]], align 4
+; CHECK-NEXT: [[V2:%.*]] = load <4 x b32>, ptr [[GEP1]], align 16
+; CHECK-NEXT: [[L33:%.*]] = extractelement <4 x b32> [[V2]], i32 0
+; CHECK-NEXT: [[BC2:%.*]] = bitcast b32 [[L33]] to i32
+; CHECK-NEXT: [[L44:%.*]] = extractelement <4 x b32> [[V2]], i32 1
+; CHECK-NEXT: [[BC3:%.*]] = bitcast b32 [[L44]] to i32
+; CHECK-NEXT: [[L55:%.*]] = extractelement <4 x b32> [[V2]], i32 2
+; CHECK-NEXT: [[BC4:%.*]] = bitcast b32 [[L55]] to i32
+; CHECK-NEXT: [[L66:%.*]] = extractelement <4 x b32> [[V2]], i32 3
+; CHECK-NEXT: [[BC5:%.*]] = bitcast b32 [[L66]] to i32
+; CHECK-NEXT: [[L77:%.*]] = extractelement <4 x b32> [[V2]], i32 3
+; CHECK-NEXT: [[BC6:%.*]] = bitcast b32 [[L77]] to i32
+; CHECK-NEXT: ret void
+;
%tmp2 = getelementptr i32, ptr %ptr, i64 1
%tmp3 = getelementptr i32, ptr %ptr, i64 2
%tmp4 = getelementptr i32, ptr %ptr, i64 3
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/vector-scalar.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/vector-scalar.ll
index 019aeea8f0a59..6d3d2ee98a213 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/vector-scalar.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/vector-scalar.ll
@@ -1,11 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -passes=load-store-vectorizer -mcpu haswell -S -o - %s | FileCheck %s
; RUN: opt -mtriple=x86_64-unknown-linux-gnu -aa-pipeline=basic-aa -passes='function(load-store-vectorizer)' -mcpu haswell -S -o - %s | FileCheck %s
; Check that the LoadStoreVectorizer does not crash due to not differentiating <1 x T> and T.
-; CHECK-LABEL: @vector_scalar(
-; CHECK: store <2 x double>
define void @vector_scalar(ptr %ptr, double %a, <1 x double> %b) {
+; CHECK-LABEL: define void @vector_scalar(
+; CHECK-SAME: ptr [[PTR:%.*]], double [[A:%.*]], <1 x double> [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast double [[A]] to b64
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x b64> poison, b64 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast <1 x double> [[B]] to b64
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x b64> [[TMP2]], b64 [[TMP3]], i32 1
+; CHECK-NEXT: store <2 x b64> [[TMP4]], ptr [[PTR]], align 8
+; CHECK-NEXT: ret void
+;
%1 = getelementptr <1 x double>, ptr %ptr, i32 1
store double %a, ptr %ptr, align 8
store <1 x double> %b, ptr %1, align 8
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add-inseltpoison.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add-inseltpoison.ll
index 6b9229e8dfd6d..42672c533c2a8 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add-inseltpoison.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add-inseltpoison.ll
@@ -15,14 +15,18 @@ define void @ld_v4i8_add_nsw(i32 %v0, i32 %v1, ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[V1:%.*]], [[TMP]]
; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
-; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP133:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[TMP184:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> poison, i8 [[TMP41]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP82]], i32 1
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP133]], i32 2
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b8>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b8 [[TMP41]] to i8
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[TMP82]] to i8
+; CHECK-NEXT: [[TMP133:%.*]] = extractelement <4 x b8> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[TMP133]] to i8
+; CHECK-NEXT: [[TMP185:%.*]] = extractelement <4 x b8> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP184:%.*]] = bitcast b8 [[TMP185]] to i8
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> poison, i8 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP3]], i32 2
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP184]], i32 3
; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]], align 4
; CHECK-NEXT: ret void
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add.ll
index cf575cef2d1ba..f9aaaa780db8e 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-i8-nested-add.ll
@@ -15,16 +15,20 @@ define void @ld_v4i8_add_nsw(i32 %v0, i32 %v1, ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[V1:%.*]], [[TMP]]
; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
-; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP133:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[TMP184:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> undef, i8 [[TMP41]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP82]], i32 1
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP133]], i32 2
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b8>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b8 [[TMP41]] to i8
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[TMP82]] to i8
+; CHECK-NEXT: [[TMP133:%.*]] = extractelement <4 x b8> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[TMP133]] to i8
+; CHECK-NEXT: [[TMP185:%.*]] = extractelement <4 x b8> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP184:%.*]] = bitcast b8 [[TMP185]] to i8
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> undef, i8 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP3]], i32 2
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP184]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]]
+; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]], align 4
; CHECK-NEXT: ret void
;
bb:
@@ -63,16 +67,20 @@ define void @ld_v4i8_add_nsw_operand_orders(i32 %v0, i32 %v1, ptr %src, ptr %dst
; CHECK-NEXT: [[TMP1:%.*]] = add nsw i32 [[V1:%.*]], [[TMP]]
; CHECK-NEXT: [[TMP2:%.*]] = sext i32 [[TMP1]] to i64
; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP2]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[TMP3]], align 1
-; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP133:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[TMP184:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
-; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> undef, i8 [[TMP41]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP82]], i32 1
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP133]], i32 2
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b8>, ptr [[TMP3]], align 1
+; CHECK-NEXT: [[TMP41:%.*]] = extractelement <4 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b8 [[TMP41]] to i8
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[TMP82]] to i8
+; CHECK-NEXT: [[TMP133:%.*]] = extractelement <4 x b8> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[TMP133]] to i8
+; CHECK-NEXT: [[TMP185:%.*]] = extractelement <4 x b8> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP184:%.*]] = bitcast b8 [[TMP185]] to i8
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> undef, i8 [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP2]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP3]], i32 2
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP184]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]]
+; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]], align 4
; CHECK-NEXT: ret void
;
bb:
@@ -116,15 +124,18 @@ define void @ld_v4i8_add_known_bits(i32 %ind0, i32 %ind1, ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[V1]], [[V0]]
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <3 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP81:%.*]] = extractelement <3 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP132:%.*]] = extractelement <3 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP183:%.*]] = extractelement <3 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP0:%.*]] = load <3 x b8>, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP81:%.*]] = extractelement <3 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b8 [[TMP81]] to i8
+; CHECK-NEXT: [[TMP132:%.*]] = extractelement <3 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[TMP132]] to i8
+; CHECK-NEXT: [[TMP183:%.*]] = extractelement <3 x b8> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[TMP183]] to i8
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> undef, i8 [[TMP4]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP81]], i32 1
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP132]], i32 2
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP183]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]]
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP2]], i32 2
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP3]], i32 3
+; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]], align 4
; CHECK-NEXT: ret void
;
bb:
@@ -165,16 +176,20 @@ define void @ld_v4i8_add_known_bits1(i32 %ind0, i32 %ind1, ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[V1]], [[V0]]
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP81:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP132:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP183:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[TMP44:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b8>, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP81:%.*]] = bitcast b8 [[TMP82]] to i8
+; CHECK-NEXT: [[TMP133:%.*]] = extractelement <4 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP132:%.*]] = bitcast b8 [[TMP133]] to i8
+; CHECK-NEXT: [[TMP184:%.*]] = extractelement <4 x b8> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP183:%.*]] = bitcast b8 [[TMP184]] to i8
+; CHECK-NEXT: [[TMP45:%.*]] = extractelement <4 x b8> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP44:%.*]] = bitcast b8 [[TMP45]] to i8
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> undef, i8 [[TMP44]], i32 0
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP81]], i32 1
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP132]], i32 2
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP183]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]]
+; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]], align 4
; CHECK-NEXT: ret void
;
bb:
@@ -221,16 +236,20 @@ define void @ld_v4i8_add_known_bits_by_assume(i32 %ind0, i32 %ind1, ptr %src, pt
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[V1]], [[V0]]
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP81:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP132:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP183:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[TMP44:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b8>, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP81:%.*]] = bitcast b8 [[TMP82]] to i8
+; CHECK-NEXT: [[TMP133:%.*]] = extractelement <4 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP132:%.*]] = bitcast b8 [[TMP133]] to i8
+; CHECK-NEXT: [[TMP184:%.*]] = extractelement <4 x b8> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP183:%.*]] = bitcast b8 [[TMP184]] to i8
+; CHECK-NEXT: [[TMP45:%.*]] = extractelement <4 x b8> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP44:%.*]] = bitcast b8 [[TMP45]] to i8
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> undef, i8 [[TMP44]], i32 0
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP81]], i32 1
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP132]], i32 2
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP183]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]]
+; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]], align 4
; CHECK-NEXT: ret void
;
bb:
@@ -288,15 +307,18 @@ define void @ld_v4i8_add_assume_on_arg(i32 %v0, i32 %v1, ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[V1]], [[V0]]
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <3 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP81:%.*]] = extractelement <3 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP132:%.*]] = extractelement <3 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP183:%.*]] = extractelement <3 x i8> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP0:%.*]] = load <3 x b8>, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP81:%.*]] = extractelement <3 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b8 [[TMP81]] to i8
+; CHECK-NEXT: [[TMP132:%.*]] = extractelement <3 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[TMP132]] to i8
+; CHECK-NEXT: [[TMP183:%.*]] = extractelement <3 x b8> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[TMP183]] to i8
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> undef, i8 [[TMP4]], i32 0
-; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP81]], i32 1
-; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP132]], i32 2
-; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP183]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]]
+; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP2]], i32 2
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP3]], i32 3
+; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]], align 4
; CHECK-NEXT: ret void
;
bb:
@@ -345,16 +367,20 @@ define void @ld_v4i8_add_assume_on_arg1(i32 %v0, i32 %v1, ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[V1]], [[V0]]
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP81:%.*]] = extractelement <4 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP132:%.*]] = extractelement <4 x i8> [[TMP1]], i32 1
-; CHECK-NEXT: [[TMP183:%.*]] = extractelement <4 x i8> [[TMP1]], i32 2
-; CHECK-NEXT: [[TMP44:%.*]] = extractelement <4 x i8> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b8>, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <4 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP81:%.*]] = bitcast b8 [[TMP82]] to i8
+; CHECK-NEXT: [[TMP133:%.*]] = extractelement <4 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP132:%.*]] = bitcast b8 [[TMP133]] to i8
+; CHECK-NEXT: [[TMP184:%.*]] = extractelement <4 x b8> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP183:%.*]] = bitcast b8 [[TMP184]] to i8
+; CHECK-NEXT: [[TMP45:%.*]] = extractelement <4 x b8> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP44:%.*]] = bitcast b8 [[TMP45]] to i8
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i8> undef, i8 [[TMP44]], i32 0
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP81]], i32 1
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP132]], i32 2
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP183]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]]
+; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]], align 4
; CHECK-NEXT: ret void
;
bb:
@@ -406,12 +432,14 @@ define void @ld_v2i8_add_different_contexts(i32 %ind0, i32 %ind1, ptr %src, ptr
; CHECK-NEXT: call void @llvm.assume(i1 [[BIT_COND]])
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP81:%.*]] = extractelement <2 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <2 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b8>, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <2 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP81:%.*]] = bitcast b8 [[TMP82]] to i8
+; CHECK-NEXT: [[TMP43:%.*]] = extractelement <2 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast b8 [[TMP43]] to i8
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i8> undef, i8 [[TMP42]], i32 0
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x i8> [[TMP19]], i8 [[TMP81]], i32 1
-; CHECK-NEXT: store <2 x i8> [[TMP20]], ptr [[DST:%.*]]
+; CHECK-NEXT: store <2 x i8> [[TMP20]], ptr [[DST:%.*]], align 2
; CHECK-NEXT: br label [[BB_SKIP]]
; CHECK: bb.skip:
; CHECK-NEXT: ret void
@@ -455,13 +483,15 @@ define void @ld_v2i8_add_different_contexts1(i32 %ind0, i32 %ind1, ptr %src, ptr
; CHECK: bb.loads:
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP81:%.*]] = extractelement <2 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <2 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b8>, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <2 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP81:%.*]] = bitcast b8 [[TMP82]] to i8
+; CHECK-NEXT: [[TMP43:%.*]] = extractelement <2 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast b8 [[TMP43]] to i8
; CHECK-NEXT: call void @llvm.assume(i1 [[BIT_COND]])
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i8> undef, i8 [[TMP42]], i32 0
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x i8> [[TMP19]], i8 [[TMP81]], i32 1
-; CHECK-NEXT: store <2 x i8> [[TMP20]], ptr [[DST:%.*]]
+; CHECK-NEXT: store <2 x i8> [[TMP20]], ptr [[DST:%.*]], align 2
; CHECK-NEXT: br label [[BB_SKIP]]
; CHECK: bb.skip:
; CHECK-NEXT: ret void
@@ -502,14 +532,16 @@ define void @ld_v2i8_add_context(i32 %ind0, i32 %ind1, ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[V1]], [[V0]]
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP81:%.*]] = extractelement <2 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <2 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b8>, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <2 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP81:%.*]] = bitcast b8 [[TMP82]] to i8
+; CHECK-NEXT: [[TMP43:%.*]] = extractelement <2 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast b8 [[TMP43]] to i8
; CHECK-NEXT: [[BIT_COND:%.*]] = icmp eq i32 [[TMP5]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[BIT_COND]])
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i8> undef, i8 [[TMP42]], i32 0
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x i8> [[TMP19]], i8 [[TMP81]], i32 1
-; CHECK-NEXT: store <2 x i8> [[TMP20]], ptr [[DST:%.*]]
+; CHECK-NEXT: store <2 x i8> [[TMP20]], ptr [[DST:%.*]], align 2
; CHECK-NEXT: ret void
;
bb:
@@ -542,12 +574,14 @@ define void @ld_v2i8_add_context1(i32 %ind0, i32 %ind1, ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP5:%.*]] = add i32 [[V1]], [[V0]]
; CHECK-NEXT: [[TMP6:%.*]] = sext i32 [[TMP5]] to i64
; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP6]]
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i8>, ptr [[TMP7]], align 1
-; CHECK-NEXT: [[TMP81:%.*]] = extractelement <2 x i8> [[TMP1]], i32 0
-; CHECK-NEXT: [[TMP42:%.*]] = extractelement <2 x i8> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x b8>, ptr [[TMP7]], align 1
+; CHECK-NEXT: [[TMP82:%.*]] = extractelement <2 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP81:%.*]] = bitcast b8 [[TMP82]] to i8
+; CHECK-NEXT: [[TMP43:%.*]] = extractelement <2 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast b8 [[TMP43]] to i8
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <2 x i8> undef, i8 [[TMP42]], i32 0
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <2 x i8> [[TMP19]], i8 [[TMP81]], i32 1
-; CHECK-NEXT: store <2 x i8> [[TMP20]], ptr [[DST:%.*]]
+; CHECK-NEXT: store <2 x i8> [[TMP20]], ptr [[DST:%.*]], align 2
; CHECK-NEXT: [[BIT_COND:%.*]] = icmp eq i32 [[TMP5]], 0
; CHECK-NEXT: call void @llvm.assume(i1 [[BIT_COND]])
; CHECK-NEXT: ret void
@@ -602,7 +636,7 @@ define void @ld_v4i8_add_not_safe(i32 %v0, i32 %v1, ptr %src, ptr %dst) {
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x i8> [[TMP19]], i8 [[TMP8]], i32 1
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x i8> [[TMP20]], i8 [[TMP13]], i32 2
; CHECK-NEXT: [[TMP22:%.*]] = insertelement <4 x i8> [[TMP21]], i8 [[TMP18]], i32 3
-; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]]
+; CHECK-NEXT: store <4 x i8> [[TMP22]], ptr [[DST:%.*]], align 4
; CHECK-NEXT: ret void
;
bb:
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-redund-loads.ll b/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-redund-loads.ll
index e2744d778fb1d..981a5a9490eec 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-redund-loads.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/X86/vectorize-redund-loads.ll
@@ -4,13 +4,17 @@
define void @test(ptr %ptr) {
; CHECK-LABEL: define void @test(
; CHECK-SAME: ptr [[PTR:%.*]]) {
-; CHECK-NEXT: [[TMP1:%.*]] = load <2 x i32>, ptr [[PTR]], align 8
-; CHECK-NEXT: [[LD01:%.*]] = extractelement <2 x i32> [[TMP1]], i32 0
-; CHECK-NEXT: [[LD12:%.*]] = extractelement <2 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x b32>, ptr [[PTR]], align 8
+; CHECK-NEXT: [[LD01:%.*]] = extractelement <2 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[LD01]] to i32
+; CHECK-NEXT: [[LD12:%.*]] = extractelement <2 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[LD12]] to i32
; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i8, ptr [[PTR]], i32 8
-; CHECK-NEXT: [[TMP2:%.*]] = load <2 x i32>, ptr [[GEP2]], align 8
-; CHECK-NEXT: [[LD23:%.*]] = shufflevector <2 x i32> [[TMP2]], <2 x i32> poison, <2 x i32> <i32 0, i32 1>
-; CHECK-NEXT: [[LD34:%.*]] = extractelement <2 x i32> [[TMP2]], i32 0
+; CHECK-NEXT: [[TMP4:%.*]] = load <2 x b32>, ptr [[GEP2]], align 8
+; CHECK-NEXT: [[LD23:%.*]] = shufflevector <2 x b32> [[TMP4]], <2 x b32> poison, <2 x i32> <i32 0, i32 1>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast <2 x b32> [[LD23]] to <2 x i32>
+; CHECK-NEXT: [[LD34:%.*]] = extractelement <2 x b32> [[TMP4]], i32 0
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast b32 [[LD34]] to i32
; CHECK-NEXT: ret void
;
%ld0 = load i32, ptr %ptr, align 8
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/batch-aa-compile-time.ll b/llvm/test/Transforms/LoadStoreVectorizer/batch-aa-compile-time.ll
index 049c9a3f70e0f..56d8fd63dbe89 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/batch-aa-compile-time.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/batch-aa-compile-time.ll
@@ -1,3 +1,4 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
; RUN: opt -S < %s -passes='loop-unroll,load-store-vectorizer' -unroll-count=128 --capture-tracking-max-uses-to-explore=1024 | FileCheck %s
; Without using batching alias analysis, this test takes 6 seconds to compile. With, less than a second.
@@ -11,7 +12,2571 @@
define void @compile-time-test() {
; CHECK-LABEL: define void @compile-time-test() {
-; CHECK-COUNT-128: load <4 x i8>
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[GLOBAL_BASE_LOADS:%.*]] = getelementptr i8, ptr inttoptr (i32 ptrtoint (ptr @global_mem to i32) to ptr), i64 0
+; CHECK-NEXT: [[LOCAL_BASE_STORES:%.*]] = alloca <512 x i8>, align 4
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x b8>, ptr [[GLOBAL_BASE_LOADS]], align 4
+; CHECK-NEXT: [[LOAD_01:%.*]] = extractelement <4 x b8> [[TMP0]], i32 0
+; CHECK-NEXT: [[TMP1:%.*]] = bitcast b8 [[LOAD_01]] to i8
+; CHECK-NEXT: [[LOAD_12:%.*]] = extractelement <4 x b8> [[TMP0]], i32 1
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b8 [[LOAD_12]] to i8
+; CHECK-NEXT: [[LOAD_23:%.*]] = extractelement <4 x b8> [[TMP0]], i32 2
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b8 [[LOAD_23]] to i8
+; CHECK-NEXT: [[LOAD_34:%.*]] = extractelement <4 x b8> [[TMP0]], i32 3
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b8 [[LOAD_34]] to i8
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i8 [[TMP1]] to b8
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x b8> poison, b8 [[TMP5]], i32 0
+; CHECK-NEXT: [[TMP7:%.*]] = bitcast i8 [[TMP2]] to b8
+; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x b8> [[TMP6]], b8 [[TMP7]], i32 1
+; CHECK-NEXT: [[TMP9:%.*]] = bitcast i8 [[TMP3]] to b8
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x b8> [[TMP8]], b8 [[TMP9]], i32 2
+; CHECK-NEXT: [[TMP11:%.*]] = bitcast i8 [[TMP4]] to b8
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x b8> [[TMP10]], b8 [[TMP11]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP12]], ptr [[LOCAL_BASE_STORES]], align 4
+; CHECK-NEXT: [[PTR_0_1:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 4
+; CHECK-NEXT: [[TMP13:%.*]] = load <4 x b8>, ptr [[PTR_0_1]], align 4
+; CHECK-NEXT: [[LOAD_0_128:%.*]] = extractelement <4 x b8> [[TMP13]], i32 0
+; CHECK-NEXT: [[TMP14:%.*]] = bitcast b8 [[LOAD_0_128]] to i8
+; CHECK-NEXT: [[LOAD_1_129:%.*]] = extractelement <4 x b8> [[TMP13]], i32 1
+; CHECK-NEXT: [[TMP15:%.*]] = bitcast b8 [[LOAD_1_129]] to i8
+; CHECK-NEXT: [[LOAD_2_130:%.*]] = extractelement <4 x b8> [[TMP13]], i32 2
+; CHECK-NEXT: [[TMP16:%.*]] = bitcast b8 [[LOAD_2_130]] to i8
+; CHECK-NEXT: [[LOAD_3_131:%.*]] = extractelement <4 x b8> [[TMP13]], i32 3
+; CHECK-NEXT: [[TMP17:%.*]] = bitcast b8 [[LOAD_3_131]] to i8
+; CHECK-NEXT: [[PTR2_0_1:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 4
+; CHECK-NEXT: [[TMP18:%.*]] = bitcast i8 [[TMP14]] to b8
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x b8> poison, b8 [[TMP18]], i32 0
+; CHECK-NEXT: [[TMP20:%.*]] = bitcast i8 [[TMP15]] to b8
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x b8> [[TMP19]], b8 [[TMP20]], i32 1
+; CHECK-NEXT: [[TMP22:%.*]] = bitcast i8 [[TMP16]] to b8
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x b8> [[TMP21]], b8 [[TMP22]], i32 2
+; CHECK-NEXT: [[TMP24:%.*]] = bitcast i8 [[TMP17]] to b8
+; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x b8> [[TMP23]], b8 [[TMP24]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP25]], ptr [[PTR2_0_1]], align 4
+; CHECK-NEXT: [[PTR_0_2:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 8
+; CHECK-NEXT: [[TMP26:%.*]] = load <4 x b8>, ptr [[PTR_0_2]], align 4
+; CHECK-NEXT: [[LOAD_0_232:%.*]] = extractelement <4 x b8> [[TMP26]], i32 0
+; CHECK-NEXT: [[TMP27:%.*]] = bitcast b8 [[LOAD_0_232]] to i8
+; CHECK-NEXT: [[LOAD_1_233:%.*]] = extractelement <4 x b8> [[TMP26]], i32 1
+; CHECK-NEXT: [[TMP28:%.*]] = bitcast b8 [[LOAD_1_233]] to i8
+; CHECK-NEXT: [[LOAD_2_234:%.*]] = extractelement <4 x b8> [[TMP26]], i32 2
+; CHECK-NEXT: [[TMP29:%.*]] = bitcast b8 [[LOAD_2_234]] to i8
+; CHECK-NEXT: [[LOAD_3_235:%.*]] = extractelement <4 x b8> [[TMP26]], i32 3
+; CHECK-NEXT: [[TMP30:%.*]] = bitcast b8 [[LOAD_3_235]] to i8
+; CHECK-NEXT: [[PTR2_0_2:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 8
+; CHECK-NEXT: [[TMP31:%.*]] = bitcast i8 [[TMP27]] to b8
+; CHECK-NEXT: [[TMP32:%.*]] = insertelement <4 x b8> poison, b8 [[TMP31]], i32 0
+; CHECK-NEXT: [[TMP33:%.*]] = bitcast i8 [[TMP28]] to b8
+; CHECK-NEXT: [[TMP34:%.*]] = insertelement <4 x b8> [[TMP32]], b8 [[TMP33]], i32 1
+; CHECK-NEXT: [[TMP35:%.*]] = bitcast i8 [[TMP29]] to b8
+; CHECK-NEXT: [[TMP36:%.*]] = insertelement <4 x b8> [[TMP34]], b8 [[TMP35]], i32 2
+; CHECK-NEXT: [[TMP37:%.*]] = bitcast i8 [[TMP30]] to b8
+; CHECK-NEXT: [[TMP38:%.*]] = insertelement <4 x b8> [[TMP36]], b8 [[TMP37]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP38]], ptr [[PTR2_0_2]], align 4
+; CHECK-NEXT: [[PTR_0_3:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 12
+; CHECK-NEXT: [[TMP39:%.*]] = load <4 x b8>, ptr [[PTR_0_3]], align 4
+; CHECK-NEXT: [[LOAD_0_336:%.*]] = extractelement <4 x b8> [[TMP39]], i32 0
+; CHECK-NEXT: [[TMP40:%.*]] = bitcast b8 [[LOAD_0_336]] to i8
+; CHECK-NEXT: [[LOAD_1_337:%.*]] = extractelement <4 x b8> [[TMP39]], i32 1
+; CHECK-NEXT: [[TMP41:%.*]] = bitcast b8 [[LOAD_1_337]] to i8
+; CHECK-NEXT: [[LOAD_2_338:%.*]] = extractelement <4 x b8> [[TMP39]], i32 2
+; CHECK-NEXT: [[TMP42:%.*]] = bitcast b8 [[LOAD_2_338]] to i8
+; CHECK-NEXT: [[LOAD_3_339:%.*]] = extractelement <4 x b8> [[TMP39]], i32 3
+; CHECK-NEXT: [[TMP43:%.*]] = bitcast b8 [[LOAD_3_339]] to i8
+; CHECK-NEXT: [[PTR2_0_3:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 12
+; CHECK-NEXT: [[TMP44:%.*]] = bitcast i8 [[TMP40]] to b8
+; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x b8> poison, b8 [[TMP44]], i32 0
+; CHECK-NEXT: [[TMP46:%.*]] = bitcast i8 [[TMP41]] to b8
+; CHECK-NEXT: [[TMP47:%.*]] = insertelement <4 x b8> [[TMP45]], b8 [[TMP46]], i32 1
+; CHECK-NEXT: [[TMP48:%.*]] = bitcast i8 [[TMP42]] to b8
+; CHECK-NEXT: [[TMP49:%.*]] = insertelement <4 x b8> [[TMP47]], b8 [[TMP48]], i32 2
+; CHECK-NEXT: [[TMP50:%.*]] = bitcast i8 [[TMP43]] to b8
+; CHECK-NEXT: [[TMP51:%.*]] = insertelement <4 x b8> [[TMP49]], b8 [[TMP50]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP51]], ptr [[PTR2_0_3]], align 4
+; CHECK-NEXT: [[PTR_0_4:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 16
+; CHECK-NEXT: [[TMP52:%.*]] = load <4 x b8>, ptr [[PTR_0_4]], align 4
+; CHECK-NEXT: [[LOAD_0_440:%.*]] = extractelement <4 x b8> [[TMP52]], i32 0
+; CHECK-NEXT: [[TMP53:%.*]] = bitcast b8 [[LOAD_0_440]] to i8
+; CHECK-NEXT: [[LOAD_1_441:%.*]] = extractelement <4 x b8> [[TMP52]], i32 1
+; CHECK-NEXT: [[TMP54:%.*]] = bitcast b8 [[LOAD_1_441]] to i8
+; CHECK-NEXT: [[LOAD_2_442:%.*]] = extractelement <4 x b8> [[TMP52]], i32 2
+; CHECK-NEXT: [[TMP55:%.*]] = bitcast b8 [[LOAD_2_442]] to i8
+; CHECK-NEXT: [[LOAD_3_443:%.*]] = extractelement <4 x b8> [[TMP52]], i32 3
+; CHECK-NEXT: [[TMP56:%.*]] = bitcast b8 [[LOAD_3_443]] to i8
+; CHECK-NEXT: [[PTR2_0_4:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 16
+; CHECK-NEXT: [[TMP57:%.*]] = bitcast i8 [[TMP53]] to b8
+; CHECK-NEXT: [[TMP58:%.*]] = insertelement <4 x b8> poison, b8 [[TMP57]], i32 0
+; CHECK-NEXT: [[TMP59:%.*]] = bitcast i8 [[TMP54]] to b8
+; CHECK-NEXT: [[TMP60:%.*]] = insertelement <4 x b8> [[TMP58]], b8 [[TMP59]], i32 1
+; CHECK-NEXT: [[TMP61:%.*]] = bitcast i8 [[TMP55]] to b8
+; CHECK-NEXT: [[TMP62:%.*]] = insertelement <4 x b8> [[TMP60]], b8 [[TMP61]], i32 2
+; CHECK-NEXT: [[TMP63:%.*]] = bitcast i8 [[TMP56]] to b8
+; CHECK-NEXT: [[TMP64:%.*]] = insertelement <4 x b8> [[TMP62]], b8 [[TMP63]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP64]], ptr [[PTR2_0_4]], align 4
+; CHECK-NEXT: [[PTR_0_5:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 20
+; CHECK-NEXT: [[TMP65:%.*]] = load <4 x b8>, ptr [[PTR_0_5]], align 4
+; CHECK-NEXT: [[LOAD_0_544:%.*]] = extractelement <4 x b8> [[TMP65]], i32 0
+; CHECK-NEXT: [[TMP66:%.*]] = bitcast b8 [[LOAD_0_544]] to i8
+; CHECK-NEXT: [[LOAD_1_545:%.*]] = extractelement <4 x b8> [[TMP65]], i32 1
+; CHECK-NEXT: [[TMP67:%.*]] = bitcast b8 [[LOAD_1_545]] to i8
+; CHECK-NEXT: [[LOAD_2_546:%.*]] = extractelement <4 x b8> [[TMP65]], i32 2
+; CHECK-NEXT: [[TMP68:%.*]] = bitcast b8 [[LOAD_2_546]] to i8
+; CHECK-NEXT: [[LOAD_3_547:%.*]] = extractelement <4 x b8> [[TMP65]], i32 3
+; CHECK-NEXT: [[TMP69:%.*]] = bitcast b8 [[LOAD_3_547]] to i8
+; CHECK-NEXT: [[PTR2_0_5:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 20
+; CHECK-NEXT: [[TMP70:%.*]] = bitcast i8 [[TMP66]] to b8
+; CHECK-NEXT: [[TMP71:%.*]] = insertelement <4 x b8> poison, b8 [[TMP70]], i32 0
+; CHECK-NEXT: [[TMP72:%.*]] = bitcast i8 [[TMP67]] to b8
+; CHECK-NEXT: [[TMP73:%.*]] = insertelement <4 x b8> [[TMP71]], b8 [[TMP72]], i32 1
+; CHECK-NEXT: [[TMP74:%.*]] = bitcast i8 [[TMP68]] to b8
+; CHECK-NEXT: [[TMP75:%.*]] = insertelement <4 x b8> [[TMP73]], b8 [[TMP74]], i32 2
+; CHECK-NEXT: [[TMP76:%.*]] = bitcast i8 [[TMP69]] to b8
+; CHECK-NEXT: [[TMP77:%.*]] = insertelement <4 x b8> [[TMP75]], b8 [[TMP76]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP77]], ptr [[PTR2_0_5]], align 4
+; CHECK-NEXT: [[PTR_0_6:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 24
+; CHECK-NEXT: [[TMP78:%.*]] = load <4 x b8>, ptr [[PTR_0_6]], align 4
+; CHECK-NEXT: [[LOAD_0_648:%.*]] = extractelement <4 x b8> [[TMP78]], i32 0
+; CHECK-NEXT: [[TMP79:%.*]] = bitcast b8 [[LOAD_0_648]] to i8
+; CHECK-NEXT: [[LOAD_1_649:%.*]] = extractelement <4 x b8> [[TMP78]], i32 1
+; CHECK-NEXT: [[TMP80:%.*]] = bitcast b8 [[LOAD_1_649]] to i8
+; CHECK-NEXT: [[LOAD_2_650:%.*]] = extractelement <4 x b8> [[TMP78]], i32 2
+; CHECK-NEXT: [[TMP81:%.*]] = bitcast b8 [[LOAD_2_650]] to i8
+; CHECK-NEXT: [[LOAD_3_651:%.*]] = extractelement <4 x b8> [[TMP78]], i32 3
+; CHECK-NEXT: [[TMP82:%.*]] = bitcast b8 [[LOAD_3_651]] to i8
+; CHECK-NEXT: [[PTR2_0_6:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 24
+; CHECK-NEXT: [[TMP83:%.*]] = bitcast i8 [[TMP79]] to b8
+; CHECK-NEXT: [[TMP84:%.*]] = insertelement <4 x b8> poison, b8 [[TMP83]], i32 0
+; CHECK-NEXT: [[TMP85:%.*]] = bitcast i8 [[TMP80]] to b8
+; CHECK-NEXT: [[TMP86:%.*]] = insertelement <4 x b8> [[TMP84]], b8 [[TMP85]], i32 1
+; CHECK-NEXT: [[TMP87:%.*]] = bitcast i8 [[TMP81]] to b8
+; CHECK-NEXT: [[TMP88:%.*]] = insertelement <4 x b8> [[TMP86]], b8 [[TMP87]], i32 2
+; CHECK-NEXT: [[TMP89:%.*]] = bitcast i8 [[TMP82]] to b8
+; CHECK-NEXT: [[TMP90:%.*]] = insertelement <4 x b8> [[TMP88]], b8 [[TMP89]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP90]], ptr [[PTR2_0_6]], align 4
+; CHECK-NEXT: [[PTR_0_7:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 28
+; CHECK-NEXT: [[TMP91:%.*]] = load <4 x b8>, ptr [[PTR_0_7]], align 4
+; CHECK-NEXT: [[LOAD_0_752:%.*]] = extractelement <4 x b8> [[TMP91]], i32 0
+; CHECK-NEXT: [[TMP92:%.*]] = bitcast b8 [[LOAD_0_752]] to i8
+; CHECK-NEXT: [[LOAD_1_753:%.*]] = extractelement <4 x b8> [[TMP91]], i32 1
+; CHECK-NEXT: [[TMP93:%.*]] = bitcast b8 [[LOAD_1_753]] to i8
+; CHECK-NEXT: [[LOAD_2_754:%.*]] = extractelement <4 x b8> [[TMP91]], i32 2
+; CHECK-NEXT: [[TMP94:%.*]] = bitcast b8 [[LOAD_2_754]] to i8
+; CHECK-NEXT: [[LOAD_3_755:%.*]] = extractelement <4 x b8> [[TMP91]], i32 3
+; CHECK-NEXT: [[TMP95:%.*]] = bitcast b8 [[LOAD_3_755]] to i8
+; CHECK-NEXT: [[PTR2_0_7:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 28
+; CHECK-NEXT: [[TMP96:%.*]] = bitcast i8 [[TMP92]] to b8
+; CHECK-NEXT: [[TMP97:%.*]] = insertelement <4 x b8> poison, b8 [[TMP96]], i32 0
+; CHECK-NEXT: [[TMP98:%.*]] = bitcast i8 [[TMP93]] to b8
+; CHECK-NEXT: [[TMP99:%.*]] = insertelement <4 x b8> [[TMP97]], b8 [[TMP98]], i32 1
+; CHECK-NEXT: [[TMP100:%.*]] = bitcast i8 [[TMP94]] to b8
+; CHECK-NEXT: [[TMP101:%.*]] = insertelement <4 x b8> [[TMP99]], b8 [[TMP100]], i32 2
+; CHECK-NEXT: [[TMP102:%.*]] = bitcast i8 [[TMP95]] to b8
+; CHECK-NEXT: [[TMP103:%.*]] = insertelement <4 x b8> [[TMP101]], b8 [[TMP102]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP103]], ptr [[PTR2_0_7]], align 4
+; CHECK-NEXT: [[PTR_0_8:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 32
+; CHECK-NEXT: [[TMP104:%.*]] = load <4 x b8>, ptr [[PTR_0_8]], align 4
+; CHECK-NEXT: [[LOAD_0_856:%.*]] = extractelement <4 x b8> [[TMP104]], i32 0
+; CHECK-NEXT: [[TMP105:%.*]] = bitcast b8 [[LOAD_0_856]] to i8
+; CHECK-NEXT: [[LOAD_1_857:%.*]] = extractelement <4 x b8> [[TMP104]], i32 1
+; CHECK-NEXT: [[TMP106:%.*]] = bitcast b8 [[LOAD_1_857]] to i8
+; CHECK-NEXT: [[LOAD_2_858:%.*]] = extractelement <4 x b8> [[TMP104]], i32 2
+; CHECK-NEXT: [[TMP107:%.*]] = bitcast b8 [[LOAD_2_858]] to i8
+; CHECK-NEXT: [[LOAD_3_859:%.*]] = extractelement <4 x b8> [[TMP104]], i32 3
+; CHECK-NEXT: [[TMP108:%.*]] = bitcast b8 [[LOAD_3_859]] to i8
+; CHECK-NEXT: [[PTR2_0_8:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 32
+; CHECK-NEXT: [[TMP109:%.*]] = bitcast i8 [[TMP105]] to b8
+; CHECK-NEXT: [[TMP110:%.*]] = insertelement <4 x b8> poison, b8 [[TMP109]], i32 0
+; CHECK-NEXT: [[TMP111:%.*]] = bitcast i8 [[TMP106]] to b8
+; CHECK-NEXT: [[TMP112:%.*]] = insertelement <4 x b8> [[TMP110]], b8 [[TMP111]], i32 1
+; CHECK-NEXT: [[TMP113:%.*]] = bitcast i8 [[TMP107]] to b8
+; CHECK-NEXT: [[TMP114:%.*]] = insertelement <4 x b8> [[TMP112]], b8 [[TMP113]], i32 2
+; CHECK-NEXT: [[TMP115:%.*]] = bitcast i8 [[TMP108]] to b8
+; CHECK-NEXT: [[TMP116:%.*]] = insertelement <4 x b8> [[TMP114]], b8 [[TMP115]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP116]], ptr [[PTR2_0_8]], align 4
+; CHECK-NEXT: [[PTR_0_9:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 36
+; CHECK-NEXT: [[TMP117:%.*]] = load <4 x b8>, ptr [[PTR_0_9]], align 4
+; CHECK-NEXT: [[LOAD_0_960:%.*]] = extractelement <4 x b8> [[TMP117]], i32 0
+; CHECK-NEXT: [[TMP118:%.*]] = bitcast b8 [[LOAD_0_960]] to i8
+; CHECK-NEXT: [[LOAD_1_961:%.*]] = extractelement <4 x b8> [[TMP117]], i32 1
+; CHECK-NEXT: [[TMP119:%.*]] = bitcast b8 [[LOAD_1_961]] to i8
+; CHECK-NEXT: [[LOAD_2_962:%.*]] = extractelement <4 x b8> [[TMP117]], i32 2
+; CHECK-NEXT: [[TMP120:%.*]] = bitcast b8 [[LOAD_2_962]] to i8
+; CHECK-NEXT: [[LOAD_3_963:%.*]] = extractelement <4 x b8> [[TMP117]], i32 3
+; CHECK-NEXT: [[TMP121:%.*]] = bitcast b8 [[LOAD_3_963]] to i8
+; CHECK-NEXT: [[PTR2_0_9:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 36
+; CHECK-NEXT: [[TMP122:%.*]] = bitcast i8 [[TMP118]] to b8
+; CHECK-NEXT: [[TMP123:%.*]] = insertelement <4 x b8> poison, b8 [[TMP122]], i32 0
+; CHECK-NEXT: [[TMP124:%.*]] = bitcast i8 [[TMP119]] to b8
+; CHECK-NEXT: [[TMP125:%.*]] = insertelement <4 x b8> [[TMP123]], b8 [[TMP124]], i32 1
+; CHECK-NEXT: [[TMP126:%.*]] = bitcast i8 [[TMP120]] to b8
+; CHECK-NEXT: [[TMP127:%.*]] = insertelement <4 x b8> [[TMP125]], b8 [[TMP126]], i32 2
+; CHECK-NEXT: [[TMP128:%.*]] = bitcast i8 [[TMP121]] to b8
+; CHECK-NEXT: [[TMP129:%.*]] = insertelement <4 x b8> [[TMP127]], b8 [[TMP128]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP129]], ptr [[PTR2_0_9]], align 4
+; CHECK-NEXT: [[PTR_0_10:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 40
+; CHECK-NEXT: [[TMP130:%.*]] = load <4 x b8>, ptr [[PTR_0_10]], align 4
+; CHECK-NEXT: [[LOAD_0_1064:%.*]] = extractelement <4 x b8> [[TMP130]], i32 0
+; CHECK-NEXT: [[TMP131:%.*]] = bitcast b8 [[LOAD_0_1064]] to i8
+; CHECK-NEXT: [[LOAD_1_1065:%.*]] = extractelement <4 x b8> [[TMP130]], i32 1
+; CHECK-NEXT: [[TMP132:%.*]] = bitcast b8 [[LOAD_1_1065]] to i8
+; CHECK-NEXT: [[LOAD_2_1066:%.*]] = extractelement <4 x b8> [[TMP130]], i32 2
+; CHECK-NEXT: [[TMP133:%.*]] = bitcast b8 [[LOAD_2_1066]] to i8
+; CHECK-NEXT: [[LOAD_3_1067:%.*]] = extractelement <4 x b8> [[TMP130]], i32 3
+; CHECK-NEXT: [[TMP134:%.*]] = bitcast b8 [[LOAD_3_1067]] to i8
+; CHECK-NEXT: [[PTR2_0_10:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 40
+; CHECK-NEXT: [[TMP135:%.*]] = bitcast i8 [[TMP131]] to b8
+; CHECK-NEXT: [[TMP136:%.*]] = insertelement <4 x b8> poison, b8 [[TMP135]], i32 0
+; CHECK-NEXT: [[TMP137:%.*]] = bitcast i8 [[TMP132]] to b8
+; CHECK-NEXT: [[TMP138:%.*]] = insertelement <4 x b8> [[TMP136]], b8 [[TMP137]], i32 1
+; CHECK-NEXT: [[TMP139:%.*]] = bitcast i8 [[TMP133]] to b8
+; CHECK-NEXT: [[TMP140:%.*]] = insertelement <4 x b8> [[TMP138]], b8 [[TMP139]], i32 2
+; CHECK-NEXT: [[TMP141:%.*]] = bitcast i8 [[TMP134]] to b8
+; CHECK-NEXT: [[TMP142:%.*]] = insertelement <4 x b8> [[TMP140]], b8 [[TMP141]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP142]], ptr [[PTR2_0_10]], align 4
+; CHECK-NEXT: [[PTR_0_11:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 44
+; CHECK-NEXT: [[TMP143:%.*]] = load <4 x b8>, ptr [[PTR_0_11]], align 4
+; CHECK-NEXT: [[LOAD_0_1168:%.*]] = extractelement <4 x b8> [[TMP143]], i32 0
+; CHECK-NEXT: [[TMP144:%.*]] = bitcast b8 [[LOAD_0_1168]] to i8
+; CHECK-NEXT: [[LOAD_1_1169:%.*]] = extractelement <4 x b8> [[TMP143]], i32 1
+; CHECK-NEXT: [[TMP145:%.*]] = bitcast b8 [[LOAD_1_1169]] to i8
+; CHECK-NEXT: [[LOAD_2_1170:%.*]] = extractelement <4 x b8> [[TMP143]], i32 2
+; CHECK-NEXT: [[TMP146:%.*]] = bitcast b8 [[LOAD_2_1170]] to i8
+; CHECK-NEXT: [[LOAD_3_1171:%.*]] = extractelement <4 x b8> [[TMP143]], i32 3
+; CHECK-NEXT: [[TMP147:%.*]] = bitcast b8 [[LOAD_3_1171]] to i8
+; CHECK-NEXT: [[PTR2_0_11:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 44
+; CHECK-NEXT: [[TMP148:%.*]] = bitcast i8 [[TMP144]] to b8
+; CHECK-NEXT: [[TMP149:%.*]] = insertelement <4 x b8> poison, b8 [[TMP148]], i32 0
+; CHECK-NEXT: [[TMP150:%.*]] = bitcast i8 [[TMP145]] to b8
+; CHECK-NEXT: [[TMP151:%.*]] = insertelement <4 x b8> [[TMP149]], b8 [[TMP150]], i32 1
+; CHECK-NEXT: [[TMP152:%.*]] = bitcast i8 [[TMP146]] to b8
+; CHECK-NEXT: [[TMP153:%.*]] = insertelement <4 x b8> [[TMP151]], b8 [[TMP152]], i32 2
+; CHECK-NEXT: [[TMP154:%.*]] = bitcast i8 [[TMP147]] to b8
+; CHECK-NEXT: [[TMP155:%.*]] = insertelement <4 x b8> [[TMP153]], b8 [[TMP154]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP155]], ptr [[PTR2_0_11]], align 4
+; CHECK-NEXT: [[PTR_0_12:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 48
+; CHECK-NEXT: [[TMP156:%.*]] = load <4 x b8>, ptr [[PTR_0_12]], align 4
+; CHECK-NEXT: [[LOAD_0_1272:%.*]] = extractelement <4 x b8> [[TMP156]], i32 0
+; CHECK-NEXT: [[TMP157:%.*]] = bitcast b8 [[LOAD_0_1272]] to i8
+; CHECK-NEXT: [[LOAD_1_1273:%.*]] = extractelement <4 x b8> [[TMP156]], i32 1
+; CHECK-NEXT: [[TMP158:%.*]] = bitcast b8 [[LOAD_1_1273]] to i8
+; CHECK-NEXT: [[LOAD_2_1274:%.*]] = extractelement <4 x b8> [[TMP156]], i32 2
+; CHECK-NEXT: [[TMP159:%.*]] = bitcast b8 [[LOAD_2_1274]] to i8
+; CHECK-NEXT: [[LOAD_3_1275:%.*]] = extractelement <4 x b8> [[TMP156]], i32 3
+; CHECK-NEXT: [[TMP160:%.*]] = bitcast b8 [[LOAD_3_1275]] to i8
+; CHECK-NEXT: [[PTR2_0_12:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 48
+; CHECK-NEXT: [[TMP161:%.*]] = bitcast i8 [[TMP157]] to b8
+; CHECK-NEXT: [[TMP162:%.*]] = insertelement <4 x b8> poison, b8 [[TMP161]], i32 0
+; CHECK-NEXT: [[TMP163:%.*]] = bitcast i8 [[TMP158]] to b8
+; CHECK-NEXT: [[TMP164:%.*]] = insertelement <4 x b8> [[TMP162]], b8 [[TMP163]], i32 1
+; CHECK-NEXT: [[TMP165:%.*]] = bitcast i8 [[TMP159]] to b8
+; CHECK-NEXT: [[TMP166:%.*]] = insertelement <4 x b8> [[TMP164]], b8 [[TMP165]], i32 2
+; CHECK-NEXT: [[TMP167:%.*]] = bitcast i8 [[TMP160]] to b8
+; CHECK-NEXT: [[TMP168:%.*]] = insertelement <4 x b8> [[TMP166]], b8 [[TMP167]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP168]], ptr [[PTR2_0_12]], align 4
+; CHECK-NEXT: [[PTR_0_13:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 52
+; CHECK-NEXT: [[TMP169:%.*]] = load <4 x b8>, ptr [[PTR_0_13]], align 4
+; CHECK-NEXT: [[LOAD_0_1376:%.*]] = extractelement <4 x b8> [[TMP169]], i32 0
+; CHECK-NEXT: [[TMP170:%.*]] = bitcast b8 [[LOAD_0_1376]] to i8
+; CHECK-NEXT: [[LOAD_1_1377:%.*]] = extractelement <4 x b8> [[TMP169]], i32 1
+; CHECK-NEXT: [[TMP171:%.*]] = bitcast b8 [[LOAD_1_1377]] to i8
+; CHECK-NEXT: [[LOAD_2_1378:%.*]] = extractelement <4 x b8> [[TMP169]], i32 2
+; CHECK-NEXT: [[TMP172:%.*]] = bitcast b8 [[LOAD_2_1378]] to i8
+; CHECK-NEXT: [[LOAD_3_1379:%.*]] = extractelement <4 x b8> [[TMP169]], i32 3
+; CHECK-NEXT: [[TMP173:%.*]] = bitcast b8 [[LOAD_3_1379]] to i8
+; CHECK-NEXT: [[PTR2_0_13:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 52
+; CHECK-NEXT: [[TMP174:%.*]] = bitcast i8 [[TMP170]] to b8
+; CHECK-NEXT: [[TMP175:%.*]] = insertelement <4 x b8> poison, b8 [[TMP174]], i32 0
+; CHECK-NEXT: [[TMP176:%.*]] = bitcast i8 [[TMP171]] to b8
+; CHECK-NEXT: [[TMP177:%.*]] = insertelement <4 x b8> [[TMP175]], b8 [[TMP176]], i32 1
+; CHECK-NEXT: [[TMP178:%.*]] = bitcast i8 [[TMP172]] to b8
+; CHECK-NEXT: [[TMP179:%.*]] = insertelement <4 x b8> [[TMP177]], b8 [[TMP178]], i32 2
+; CHECK-NEXT: [[TMP180:%.*]] = bitcast i8 [[TMP173]] to b8
+; CHECK-NEXT: [[TMP181:%.*]] = insertelement <4 x b8> [[TMP179]], b8 [[TMP180]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP181]], ptr [[PTR2_0_13]], align 4
+; CHECK-NEXT: [[PTR_0_14:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 56
+; CHECK-NEXT: [[TMP182:%.*]] = load <4 x b8>, ptr [[PTR_0_14]], align 4
+; CHECK-NEXT: [[LOAD_0_1480:%.*]] = extractelement <4 x b8> [[TMP182]], i32 0
+; CHECK-NEXT: [[TMP183:%.*]] = bitcast b8 [[LOAD_0_1480]] to i8
+; CHECK-NEXT: [[LOAD_1_1481:%.*]] = extractelement <4 x b8> [[TMP182]], i32 1
+; CHECK-NEXT: [[TMP184:%.*]] = bitcast b8 [[LOAD_1_1481]] to i8
+; CHECK-NEXT: [[LOAD_2_1482:%.*]] = extractelement <4 x b8> [[TMP182]], i32 2
+; CHECK-NEXT: [[TMP185:%.*]] = bitcast b8 [[LOAD_2_1482]] to i8
+; CHECK-NEXT: [[LOAD_3_1483:%.*]] = extractelement <4 x b8> [[TMP182]], i32 3
+; CHECK-NEXT: [[TMP186:%.*]] = bitcast b8 [[LOAD_3_1483]] to i8
+; CHECK-NEXT: [[PTR2_0_14:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 56
+; CHECK-NEXT: [[TMP187:%.*]] = bitcast i8 [[TMP183]] to b8
+; CHECK-NEXT: [[TMP188:%.*]] = insertelement <4 x b8> poison, b8 [[TMP187]], i32 0
+; CHECK-NEXT: [[TMP189:%.*]] = bitcast i8 [[TMP184]] to b8
+; CHECK-NEXT: [[TMP190:%.*]] = insertelement <4 x b8> [[TMP188]], b8 [[TMP189]], i32 1
+; CHECK-NEXT: [[TMP191:%.*]] = bitcast i8 [[TMP185]] to b8
+; CHECK-NEXT: [[TMP192:%.*]] = insertelement <4 x b8> [[TMP190]], b8 [[TMP191]], i32 2
+; CHECK-NEXT: [[TMP193:%.*]] = bitcast i8 [[TMP186]] to b8
+; CHECK-NEXT: [[TMP194:%.*]] = insertelement <4 x b8> [[TMP192]], b8 [[TMP193]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP194]], ptr [[PTR2_0_14]], align 4
+; CHECK-NEXT: [[PTR_0_15:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 60
+; CHECK-NEXT: [[TMP195:%.*]] = load <4 x b8>, ptr [[PTR_0_15]], align 4
+; CHECK-NEXT: [[LOAD_0_1584:%.*]] = extractelement <4 x b8> [[TMP195]], i32 0
+; CHECK-NEXT: [[TMP196:%.*]] = bitcast b8 [[LOAD_0_1584]] to i8
+; CHECK-NEXT: [[LOAD_1_1585:%.*]] = extractelement <4 x b8> [[TMP195]], i32 1
+; CHECK-NEXT: [[TMP197:%.*]] = bitcast b8 [[LOAD_1_1585]] to i8
+; CHECK-NEXT: [[LOAD_2_1586:%.*]] = extractelement <4 x b8> [[TMP195]], i32 2
+; CHECK-NEXT: [[TMP198:%.*]] = bitcast b8 [[LOAD_2_1586]] to i8
+; CHECK-NEXT: [[LOAD_3_1587:%.*]] = extractelement <4 x b8> [[TMP195]], i32 3
+; CHECK-NEXT: [[TMP199:%.*]] = bitcast b8 [[LOAD_3_1587]] to i8
+; CHECK-NEXT: [[PTR2_0_15:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 60
+; CHECK-NEXT: [[TMP200:%.*]] = bitcast i8 [[TMP196]] to b8
+; CHECK-NEXT: [[TMP201:%.*]] = insertelement <4 x b8> poison, b8 [[TMP200]], i32 0
+; CHECK-NEXT: [[TMP202:%.*]] = bitcast i8 [[TMP197]] to b8
+; CHECK-NEXT: [[TMP203:%.*]] = insertelement <4 x b8> [[TMP201]], b8 [[TMP202]], i32 1
+; CHECK-NEXT: [[TMP204:%.*]] = bitcast i8 [[TMP198]] to b8
+; CHECK-NEXT: [[TMP205:%.*]] = insertelement <4 x b8> [[TMP203]], b8 [[TMP204]], i32 2
+; CHECK-NEXT: [[TMP206:%.*]] = bitcast i8 [[TMP199]] to b8
+; CHECK-NEXT: [[TMP207:%.*]] = insertelement <4 x b8> [[TMP205]], b8 [[TMP206]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP207]], ptr [[PTR2_0_15]], align 4
+; CHECK-NEXT: [[PTR_0_16:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 64
+; CHECK-NEXT: [[TMP208:%.*]] = load <4 x b8>, ptr [[PTR_0_16]], align 4
+; CHECK-NEXT: [[LOAD_0_1688:%.*]] = extractelement <4 x b8> [[TMP208]], i32 0
+; CHECK-NEXT: [[TMP209:%.*]] = bitcast b8 [[LOAD_0_1688]] to i8
+; CHECK-NEXT: [[LOAD_1_1689:%.*]] = extractelement <4 x b8> [[TMP208]], i32 1
+; CHECK-NEXT: [[TMP210:%.*]] = bitcast b8 [[LOAD_1_1689]] to i8
+; CHECK-NEXT: [[LOAD_2_1690:%.*]] = extractelement <4 x b8> [[TMP208]], i32 2
+; CHECK-NEXT: [[TMP211:%.*]] = bitcast b8 [[LOAD_2_1690]] to i8
+; CHECK-NEXT: [[LOAD_3_1691:%.*]] = extractelement <4 x b8> [[TMP208]], i32 3
+; CHECK-NEXT: [[TMP212:%.*]] = bitcast b8 [[LOAD_3_1691]] to i8
+; CHECK-NEXT: [[PTR2_0_16:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 64
+; CHECK-NEXT: [[TMP213:%.*]] = bitcast i8 [[TMP209]] to b8
+; CHECK-NEXT: [[TMP214:%.*]] = insertelement <4 x b8> poison, b8 [[TMP213]], i32 0
+; CHECK-NEXT: [[TMP215:%.*]] = bitcast i8 [[TMP210]] to b8
+; CHECK-NEXT: [[TMP216:%.*]] = insertelement <4 x b8> [[TMP214]], b8 [[TMP215]], i32 1
+; CHECK-NEXT: [[TMP217:%.*]] = bitcast i8 [[TMP211]] to b8
+; CHECK-NEXT: [[TMP218:%.*]] = insertelement <4 x b8> [[TMP216]], b8 [[TMP217]], i32 2
+; CHECK-NEXT: [[TMP219:%.*]] = bitcast i8 [[TMP212]] to b8
+; CHECK-NEXT: [[TMP220:%.*]] = insertelement <4 x b8> [[TMP218]], b8 [[TMP219]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP220]], ptr [[PTR2_0_16]], align 4
+; CHECK-NEXT: [[PTR_0_17:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 68
+; CHECK-NEXT: [[TMP221:%.*]] = load <4 x b8>, ptr [[PTR_0_17]], align 4
+; CHECK-NEXT: [[LOAD_0_1792:%.*]] = extractelement <4 x b8> [[TMP221]], i32 0
+; CHECK-NEXT: [[TMP222:%.*]] = bitcast b8 [[LOAD_0_1792]] to i8
+; CHECK-NEXT: [[LOAD_1_1793:%.*]] = extractelement <4 x b8> [[TMP221]], i32 1
+; CHECK-NEXT: [[TMP223:%.*]] = bitcast b8 [[LOAD_1_1793]] to i8
+; CHECK-NEXT: [[LOAD_2_1794:%.*]] = extractelement <4 x b8> [[TMP221]], i32 2
+; CHECK-NEXT: [[TMP224:%.*]] = bitcast b8 [[LOAD_2_1794]] to i8
+; CHECK-NEXT: [[LOAD_3_1795:%.*]] = extractelement <4 x b8> [[TMP221]], i32 3
+; CHECK-NEXT: [[TMP225:%.*]] = bitcast b8 [[LOAD_3_1795]] to i8
+; CHECK-NEXT: [[PTR2_0_17:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 68
+; CHECK-NEXT: [[TMP226:%.*]] = bitcast i8 [[TMP222]] to b8
+; CHECK-NEXT: [[TMP227:%.*]] = insertelement <4 x b8> poison, b8 [[TMP226]], i32 0
+; CHECK-NEXT: [[TMP228:%.*]] = bitcast i8 [[TMP223]] to b8
+; CHECK-NEXT: [[TMP229:%.*]] = insertelement <4 x b8> [[TMP227]], b8 [[TMP228]], i32 1
+; CHECK-NEXT: [[TMP230:%.*]] = bitcast i8 [[TMP224]] to b8
+; CHECK-NEXT: [[TMP231:%.*]] = insertelement <4 x b8> [[TMP229]], b8 [[TMP230]], i32 2
+; CHECK-NEXT: [[TMP232:%.*]] = bitcast i8 [[TMP225]] to b8
+; CHECK-NEXT: [[TMP233:%.*]] = insertelement <4 x b8> [[TMP231]], b8 [[TMP232]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP233]], ptr [[PTR2_0_17]], align 4
+; CHECK-NEXT: [[PTR_0_18:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 72
+; CHECK-NEXT: [[TMP234:%.*]] = load <4 x b8>, ptr [[PTR_0_18]], align 4
+; CHECK-NEXT: [[LOAD_0_1896:%.*]] = extractelement <4 x b8> [[TMP234]], i32 0
+; CHECK-NEXT: [[TMP235:%.*]] = bitcast b8 [[LOAD_0_1896]] to i8
+; CHECK-NEXT: [[LOAD_1_1897:%.*]] = extractelement <4 x b8> [[TMP234]], i32 1
+; CHECK-NEXT: [[TMP236:%.*]] = bitcast b8 [[LOAD_1_1897]] to i8
+; CHECK-NEXT: [[LOAD_2_1898:%.*]] = extractelement <4 x b8> [[TMP234]], i32 2
+; CHECK-NEXT: [[TMP237:%.*]] = bitcast b8 [[LOAD_2_1898]] to i8
+; CHECK-NEXT: [[LOAD_3_1899:%.*]] = extractelement <4 x b8> [[TMP234]], i32 3
+; CHECK-NEXT: [[TMP238:%.*]] = bitcast b8 [[LOAD_3_1899]] to i8
+; CHECK-NEXT: [[PTR2_0_18:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 72
+; CHECK-NEXT: [[TMP239:%.*]] = bitcast i8 [[TMP235]] to b8
+; CHECK-NEXT: [[TMP240:%.*]] = insertelement <4 x b8> poison, b8 [[TMP239]], i32 0
+; CHECK-NEXT: [[TMP241:%.*]] = bitcast i8 [[TMP236]] to b8
+; CHECK-NEXT: [[TMP242:%.*]] = insertelement <4 x b8> [[TMP240]], b8 [[TMP241]], i32 1
+; CHECK-NEXT: [[TMP243:%.*]] = bitcast i8 [[TMP237]] to b8
+; CHECK-NEXT: [[TMP244:%.*]] = insertelement <4 x b8> [[TMP242]], b8 [[TMP243]], i32 2
+; CHECK-NEXT: [[TMP245:%.*]] = bitcast i8 [[TMP238]] to b8
+; CHECK-NEXT: [[TMP246:%.*]] = insertelement <4 x b8> [[TMP244]], b8 [[TMP245]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP246]], ptr [[PTR2_0_18]], align 4
+; CHECK-NEXT: [[PTR_0_19:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 76
+; CHECK-NEXT: [[TMP247:%.*]] = load <4 x b8>, ptr [[PTR_0_19]], align 4
+; CHECK-NEXT: [[LOAD_0_19100:%.*]] = extractelement <4 x b8> [[TMP247]], i32 0
+; CHECK-NEXT: [[TMP248:%.*]] = bitcast b8 [[LOAD_0_19100]] to i8
+; CHECK-NEXT: [[LOAD_1_19101:%.*]] = extractelement <4 x b8> [[TMP247]], i32 1
+; CHECK-NEXT: [[TMP249:%.*]] = bitcast b8 [[LOAD_1_19101]] to i8
+; CHECK-NEXT: [[LOAD_2_19102:%.*]] = extractelement <4 x b8> [[TMP247]], i32 2
+; CHECK-NEXT: [[TMP250:%.*]] = bitcast b8 [[LOAD_2_19102]] to i8
+; CHECK-NEXT: [[LOAD_3_19103:%.*]] = extractelement <4 x b8> [[TMP247]], i32 3
+; CHECK-NEXT: [[TMP251:%.*]] = bitcast b8 [[LOAD_3_19103]] to i8
+; CHECK-NEXT: [[PTR2_0_19:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 76
+; CHECK-NEXT: [[TMP252:%.*]] = bitcast i8 [[TMP248]] to b8
+; CHECK-NEXT: [[TMP253:%.*]] = insertelement <4 x b8> poison, b8 [[TMP252]], i32 0
+; CHECK-NEXT: [[TMP254:%.*]] = bitcast i8 [[TMP249]] to b8
+; CHECK-NEXT: [[TMP255:%.*]] = insertelement <4 x b8> [[TMP253]], b8 [[TMP254]], i32 1
+; CHECK-NEXT: [[TMP256:%.*]] = bitcast i8 [[TMP250]] to b8
+; CHECK-NEXT: [[TMP257:%.*]] = insertelement <4 x b8> [[TMP255]], b8 [[TMP256]], i32 2
+; CHECK-NEXT: [[TMP258:%.*]] = bitcast i8 [[TMP251]] to b8
+; CHECK-NEXT: [[TMP259:%.*]] = insertelement <4 x b8> [[TMP257]], b8 [[TMP258]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP259]], ptr [[PTR2_0_19]], align 4
+; CHECK-NEXT: [[PTR_0_20:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 80
+; CHECK-NEXT: [[TMP260:%.*]] = load <4 x b8>, ptr [[PTR_0_20]], align 4
+; CHECK-NEXT: [[LOAD_0_20104:%.*]] = extractelement <4 x b8> [[TMP260]], i32 0
+; CHECK-NEXT: [[TMP261:%.*]] = bitcast b8 [[LOAD_0_20104]] to i8
+; CHECK-NEXT: [[LOAD_1_20105:%.*]] = extractelement <4 x b8> [[TMP260]], i32 1
+; CHECK-NEXT: [[TMP262:%.*]] = bitcast b8 [[LOAD_1_20105]] to i8
+; CHECK-NEXT: [[LOAD_2_20106:%.*]] = extractelement <4 x b8> [[TMP260]], i32 2
+; CHECK-NEXT: [[TMP263:%.*]] = bitcast b8 [[LOAD_2_20106]] to i8
+; CHECK-NEXT: [[LOAD_3_20107:%.*]] = extractelement <4 x b8> [[TMP260]], i32 3
+; CHECK-NEXT: [[TMP264:%.*]] = bitcast b8 [[LOAD_3_20107]] to i8
+; CHECK-NEXT: [[PTR2_0_20:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 80
+; CHECK-NEXT: [[TMP265:%.*]] = bitcast i8 [[TMP261]] to b8
+; CHECK-NEXT: [[TMP266:%.*]] = insertelement <4 x b8> poison, b8 [[TMP265]], i32 0
+; CHECK-NEXT: [[TMP267:%.*]] = bitcast i8 [[TMP262]] to b8
+; CHECK-NEXT: [[TMP268:%.*]] = insertelement <4 x b8> [[TMP266]], b8 [[TMP267]], i32 1
+; CHECK-NEXT: [[TMP269:%.*]] = bitcast i8 [[TMP263]] to b8
+; CHECK-NEXT: [[TMP270:%.*]] = insertelement <4 x b8> [[TMP268]], b8 [[TMP269]], i32 2
+; CHECK-NEXT: [[TMP271:%.*]] = bitcast i8 [[TMP264]] to b8
+; CHECK-NEXT: [[TMP272:%.*]] = insertelement <4 x b8> [[TMP270]], b8 [[TMP271]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP272]], ptr [[PTR2_0_20]], align 4
+; CHECK-NEXT: [[PTR_0_21:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 84
+; CHECK-NEXT: [[TMP273:%.*]] = load <4 x b8>, ptr [[PTR_0_21]], align 4
+; CHECK-NEXT: [[LOAD_0_21108:%.*]] = extractelement <4 x b8> [[TMP273]], i32 0
+; CHECK-NEXT: [[TMP274:%.*]] = bitcast b8 [[LOAD_0_21108]] to i8
+; CHECK-NEXT: [[LOAD_1_21109:%.*]] = extractelement <4 x b8> [[TMP273]], i32 1
+; CHECK-NEXT: [[TMP275:%.*]] = bitcast b8 [[LOAD_1_21109]] to i8
+; CHECK-NEXT: [[LOAD_2_21110:%.*]] = extractelement <4 x b8> [[TMP273]], i32 2
+; CHECK-NEXT: [[TMP276:%.*]] = bitcast b8 [[LOAD_2_21110]] to i8
+; CHECK-NEXT: [[LOAD_3_21111:%.*]] = extractelement <4 x b8> [[TMP273]], i32 3
+; CHECK-NEXT: [[TMP277:%.*]] = bitcast b8 [[LOAD_3_21111]] to i8
+; CHECK-NEXT: [[PTR2_0_21:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 84
+; CHECK-NEXT: [[TMP278:%.*]] = bitcast i8 [[TMP274]] to b8
+; CHECK-NEXT: [[TMP279:%.*]] = insertelement <4 x b8> poison, b8 [[TMP278]], i32 0
+; CHECK-NEXT: [[TMP280:%.*]] = bitcast i8 [[TMP275]] to b8
+; CHECK-NEXT: [[TMP281:%.*]] = insertelement <4 x b8> [[TMP279]], b8 [[TMP280]], i32 1
+; CHECK-NEXT: [[TMP282:%.*]] = bitcast i8 [[TMP276]] to b8
+; CHECK-NEXT: [[TMP283:%.*]] = insertelement <4 x b8> [[TMP281]], b8 [[TMP282]], i32 2
+; CHECK-NEXT: [[TMP284:%.*]] = bitcast i8 [[TMP277]] to b8
+; CHECK-NEXT: [[TMP285:%.*]] = insertelement <4 x b8> [[TMP283]], b8 [[TMP284]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP285]], ptr [[PTR2_0_21]], align 4
+; CHECK-NEXT: [[PTR_0_22:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 88
+; CHECK-NEXT: [[TMP286:%.*]] = load <4 x b8>, ptr [[PTR_0_22]], align 4
+; CHECK-NEXT: [[LOAD_0_22112:%.*]] = extractelement <4 x b8> [[TMP286]], i32 0
+; CHECK-NEXT: [[TMP287:%.*]] = bitcast b8 [[LOAD_0_22112]] to i8
+; CHECK-NEXT: [[LOAD_1_22113:%.*]] = extractelement <4 x b8> [[TMP286]], i32 1
+; CHECK-NEXT: [[TMP288:%.*]] = bitcast b8 [[LOAD_1_22113]] to i8
+; CHECK-NEXT: [[LOAD_2_22114:%.*]] = extractelement <4 x b8> [[TMP286]], i32 2
+; CHECK-NEXT: [[TMP289:%.*]] = bitcast b8 [[LOAD_2_22114]] to i8
+; CHECK-NEXT: [[LOAD_3_22115:%.*]] = extractelement <4 x b8> [[TMP286]], i32 3
+; CHECK-NEXT: [[TMP290:%.*]] = bitcast b8 [[LOAD_3_22115]] to i8
+; CHECK-NEXT: [[PTR2_0_22:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 88
+; CHECK-NEXT: [[TMP291:%.*]] = bitcast i8 [[TMP287]] to b8
+; CHECK-NEXT: [[TMP292:%.*]] = insertelement <4 x b8> poison, b8 [[TMP291]], i32 0
+; CHECK-NEXT: [[TMP293:%.*]] = bitcast i8 [[TMP288]] to b8
+; CHECK-NEXT: [[TMP294:%.*]] = insertelement <4 x b8> [[TMP292]], b8 [[TMP293]], i32 1
+; CHECK-NEXT: [[TMP295:%.*]] = bitcast i8 [[TMP289]] to b8
+; CHECK-NEXT: [[TMP296:%.*]] = insertelement <4 x b8> [[TMP294]], b8 [[TMP295]], i32 2
+; CHECK-NEXT: [[TMP297:%.*]] = bitcast i8 [[TMP290]] to b8
+; CHECK-NEXT: [[TMP298:%.*]] = insertelement <4 x b8> [[TMP296]], b8 [[TMP297]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP298]], ptr [[PTR2_0_22]], align 4
+; CHECK-NEXT: [[PTR_0_23:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 92
+; CHECK-NEXT: [[TMP299:%.*]] = load <4 x b8>, ptr [[PTR_0_23]], align 4
+; CHECK-NEXT: [[LOAD_0_23116:%.*]] = extractelement <4 x b8> [[TMP299]], i32 0
+; CHECK-NEXT: [[TMP300:%.*]] = bitcast b8 [[LOAD_0_23116]] to i8
+; CHECK-NEXT: [[LOAD_1_23117:%.*]] = extractelement <4 x b8> [[TMP299]], i32 1
+; CHECK-NEXT: [[TMP301:%.*]] = bitcast b8 [[LOAD_1_23117]] to i8
+; CHECK-NEXT: [[LOAD_2_23118:%.*]] = extractelement <4 x b8> [[TMP299]], i32 2
+; CHECK-NEXT: [[TMP302:%.*]] = bitcast b8 [[LOAD_2_23118]] to i8
+; CHECK-NEXT: [[LOAD_3_23119:%.*]] = extractelement <4 x b8> [[TMP299]], i32 3
+; CHECK-NEXT: [[TMP303:%.*]] = bitcast b8 [[LOAD_3_23119]] to i8
+; CHECK-NEXT: [[PTR2_0_23:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 92
+; CHECK-NEXT: [[TMP304:%.*]] = bitcast i8 [[TMP300]] to b8
+; CHECK-NEXT: [[TMP305:%.*]] = insertelement <4 x b8> poison, b8 [[TMP304]], i32 0
+; CHECK-NEXT: [[TMP306:%.*]] = bitcast i8 [[TMP301]] to b8
+; CHECK-NEXT: [[TMP307:%.*]] = insertelement <4 x b8> [[TMP305]], b8 [[TMP306]], i32 1
+; CHECK-NEXT: [[TMP308:%.*]] = bitcast i8 [[TMP302]] to b8
+; CHECK-NEXT: [[TMP309:%.*]] = insertelement <4 x b8> [[TMP307]], b8 [[TMP308]], i32 2
+; CHECK-NEXT: [[TMP310:%.*]] = bitcast i8 [[TMP303]] to b8
+; CHECK-NEXT: [[TMP311:%.*]] = insertelement <4 x b8> [[TMP309]], b8 [[TMP310]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP311]], ptr [[PTR2_0_23]], align 4
+; CHECK-NEXT: [[PTR_0_24:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 96
+; CHECK-NEXT: [[TMP312:%.*]] = load <4 x b8>, ptr [[PTR_0_24]], align 4
+; CHECK-NEXT: [[LOAD_0_24120:%.*]] = extractelement <4 x b8> [[TMP312]], i32 0
+; CHECK-NEXT: [[TMP313:%.*]] = bitcast b8 [[LOAD_0_24120]] to i8
+; CHECK-NEXT: [[LOAD_1_24121:%.*]] = extractelement <4 x b8> [[TMP312]], i32 1
+; CHECK-NEXT: [[TMP314:%.*]] = bitcast b8 [[LOAD_1_24121]] to i8
+; CHECK-NEXT: [[LOAD_2_24122:%.*]] = extractelement <4 x b8> [[TMP312]], i32 2
+; CHECK-NEXT: [[TMP315:%.*]] = bitcast b8 [[LOAD_2_24122]] to i8
+; CHECK-NEXT: [[LOAD_3_24123:%.*]] = extractelement <4 x b8> [[TMP312]], i32 3
+; CHECK-NEXT: [[TMP316:%.*]] = bitcast b8 [[LOAD_3_24123]] to i8
+; CHECK-NEXT: [[PTR2_0_24:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 96
+; CHECK-NEXT: [[TMP317:%.*]] = bitcast i8 [[TMP313]] to b8
+; CHECK-NEXT: [[TMP318:%.*]] = insertelement <4 x b8> poison, b8 [[TMP317]], i32 0
+; CHECK-NEXT: [[TMP319:%.*]] = bitcast i8 [[TMP314]] to b8
+; CHECK-NEXT: [[TMP320:%.*]] = insertelement <4 x b8> [[TMP318]], b8 [[TMP319]], i32 1
+; CHECK-NEXT: [[TMP321:%.*]] = bitcast i8 [[TMP315]] to b8
+; CHECK-NEXT: [[TMP322:%.*]] = insertelement <4 x b8> [[TMP320]], b8 [[TMP321]], i32 2
+; CHECK-NEXT: [[TMP323:%.*]] = bitcast i8 [[TMP316]] to b8
+; CHECK-NEXT: [[TMP324:%.*]] = insertelement <4 x b8> [[TMP322]], b8 [[TMP323]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP324]], ptr [[PTR2_0_24]], align 4
+; CHECK-NEXT: [[PTR_0_25:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 100
+; CHECK-NEXT: [[TMP325:%.*]] = load <4 x b8>, ptr [[PTR_0_25]], align 4
+; CHECK-NEXT: [[LOAD_0_25124:%.*]] = extractelement <4 x b8> [[TMP325]], i32 0
+; CHECK-NEXT: [[TMP326:%.*]] = bitcast b8 [[LOAD_0_25124]] to i8
+; CHECK-NEXT: [[LOAD_1_25125:%.*]] = extractelement <4 x b8> [[TMP325]], i32 1
+; CHECK-NEXT: [[TMP327:%.*]] = bitcast b8 [[LOAD_1_25125]] to i8
+; CHECK-NEXT: [[LOAD_2_25126:%.*]] = extractelement <4 x b8> [[TMP325]], i32 2
+; CHECK-NEXT: [[TMP328:%.*]] = bitcast b8 [[LOAD_2_25126]] to i8
+; CHECK-NEXT: [[LOAD_3_25127:%.*]] = extractelement <4 x b8> [[TMP325]], i32 3
+; CHECK-NEXT: [[TMP329:%.*]] = bitcast b8 [[LOAD_3_25127]] to i8
+; CHECK-NEXT: [[PTR2_0_25:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 100
+; CHECK-NEXT: [[TMP330:%.*]] = bitcast i8 [[TMP326]] to b8
+; CHECK-NEXT: [[TMP331:%.*]] = insertelement <4 x b8> poison, b8 [[TMP330]], i32 0
+; CHECK-NEXT: [[TMP332:%.*]] = bitcast i8 [[TMP327]] to b8
+; CHECK-NEXT: [[TMP333:%.*]] = insertelement <4 x b8> [[TMP331]], b8 [[TMP332]], i32 1
+; CHECK-NEXT: [[TMP334:%.*]] = bitcast i8 [[TMP328]] to b8
+; CHECK-NEXT: [[TMP335:%.*]] = insertelement <4 x b8> [[TMP333]], b8 [[TMP334]], i32 2
+; CHECK-NEXT: [[TMP336:%.*]] = bitcast i8 [[TMP329]] to b8
+; CHECK-NEXT: [[TMP337:%.*]] = insertelement <4 x b8> [[TMP335]], b8 [[TMP336]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP337]], ptr [[PTR2_0_25]], align 4
+; CHECK-NEXT: [[PTR_0_26:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 104
+; CHECK-NEXT: [[TMP338:%.*]] = load <4 x b8>, ptr [[PTR_0_26]], align 4
+; CHECK-NEXT: [[LOAD_0_26128:%.*]] = extractelement <4 x b8> [[TMP338]], i32 0
+; CHECK-NEXT: [[TMP339:%.*]] = bitcast b8 [[LOAD_0_26128]] to i8
+; CHECK-NEXT: [[LOAD_1_26129:%.*]] = extractelement <4 x b8> [[TMP338]], i32 1
+; CHECK-NEXT: [[TMP340:%.*]] = bitcast b8 [[LOAD_1_26129]] to i8
+; CHECK-NEXT: [[LOAD_2_26130:%.*]] = extractelement <4 x b8> [[TMP338]], i32 2
+; CHECK-NEXT: [[TMP341:%.*]] = bitcast b8 [[LOAD_2_26130]] to i8
+; CHECK-NEXT: [[LOAD_3_26131:%.*]] = extractelement <4 x b8> [[TMP338]], i32 3
+; CHECK-NEXT: [[TMP342:%.*]] = bitcast b8 [[LOAD_3_26131]] to i8
+; CHECK-NEXT: [[PTR2_0_26:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 104
+; CHECK-NEXT: [[TMP343:%.*]] = bitcast i8 [[TMP339]] to b8
+; CHECK-NEXT: [[TMP344:%.*]] = insertelement <4 x b8> poison, b8 [[TMP343]], i32 0
+; CHECK-NEXT: [[TMP345:%.*]] = bitcast i8 [[TMP340]] to b8
+; CHECK-NEXT: [[TMP346:%.*]] = insertelement <4 x b8> [[TMP344]], b8 [[TMP345]], i32 1
+; CHECK-NEXT: [[TMP347:%.*]] = bitcast i8 [[TMP341]] to b8
+; CHECK-NEXT: [[TMP348:%.*]] = insertelement <4 x b8> [[TMP346]], b8 [[TMP347]], i32 2
+; CHECK-NEXT: [[TMP349:%.*]] = bitcast i8 [[TMP342]] to b8
+; CHECK-NEXT: [[TMP350:%.*]] = insertelement <4 x b8> [[TMP348]], b8 [[TMP349]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP350]], ptr [[PTR2_0_26]], align 4
+; CHECK-NEXT: [[PTR_0_27:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 108
+; CHECK-NEXT: [[TMP351:%.*]] = load <4 x b8>, ptr [[PTR_0_27]], align 4
+; CHECK-NEXT: [[LOAD_0_27132:%.*]] = extractelement <4 x b8> [[TMP351]], i32 0
+; CHECK-NEXT: [[TMP352:%.*]] = bitcast b8 [[LOAD_0_27132]] to i8
+; CHECK-NEXT: [[LOAD_1_27133:%.*]] = extractelement <4 x b8> [[TMP351]], i32 1
+; CHECK-NEXT: [[TMP353:%.*]] = bitcast b8 [[LOAD_1_27133]] to i8
+; CHECK-NEXT: [[LOAD_2_27134:%.*]] = extractelement <4 x b8> [[TMP351]], i32 2
+; CHECK-NEXT: [[TMP354:%.*]] = bitcast b8 [[LOAD_2_27134]] to i8
+; CHECK-NEXT: [[LOAD_3_27135:%.*]] = extractelement <4 x b8> [[TMP351]], i32 3
+; CHECK-NEXT: [[TMP355:%.*]] = bitcast b8 [[LOAD_3_27135]] to i8
+; CHECK-NEXT: [[PTR2_0_27:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 108
+; CHECK-NEXT: [[TMP356:%.*]] = bitcast i8 [[TMP352]] to b8
+; CHECK-NEXT: [[TMP357:%.*]] = insertelement <4 x b8> poison, b8 [[TMP356]], i32 0
+; CHECK-NEXT: [[TMP358:%.*]] = bitcast i8 [[TMP353]] to b8
+; CHECK-NEXT: [[TMP359:%.*]] = insertelement <4 x b8> [[TMP357]], b8 [[TMP358]], i32 1
+; CHECK-NEXT: [[TMP360:%.*]] = bitcast i8 [[TMP354]] to b8
+; CHECK-NEXT: [[TMP361:%.*]] = insertelement <4 x b8> [[TMP359]], b8 [[TMP360]], i32 2
+; CHECK-NEXT: [[TMP362:%.*]] = bitcast i8 [[TMP355]] to b8
+; CHECK-NEXT: [[TMP363:%.*]] = insertelement <4 x b8> [[TMP361]], b8 [[TMP362]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP363]], ptr [[PTR2_0_27]], align 4
+; CHECK-NEXT: [[PTR_0_28:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 112
+; CHECK-NEXT: [[TMP364:%.*]] = load <4 x b8>, ptr [[PTR_0_28]], align 4
+; CHECK-NEXT: [[LOAD_0_28136:%.*]] = extractelement <4 x b8> [[TMP364]], i32 0
+; CHECK-NEXT: [[TMP365:%.*]] = bitcast b8 [[LOAD_0_28136]] to i8
+; CHECK-NEXT: [[LOAD_1_28137:%.*]] = extractelement <4 x b8> [[TMP364]], i32 1
+; CHECK-NEXT: [[TMP366:%.*]] = bitcast b8 [[LOAD_1_28137]] to i8
+; CHECK-NEXT: [[LOAD_2_28138:%.*]] = extractelement <4 x b8> [[TMP364]], i32 2
+; CHECK-NEXT: [[TMP367:%.*]] = bitcast b8 [[LOAD_2_28138]] to i8
+; CHECK-NEXT: [[LOAD_3_28139:%.*]] = extractelement <4 x b8> [[TMP364]], i32 3
+; CHECK-NEXT: [[TMP368:%.*]] = bitcast b8 [[LOAD_3_28139]] to i8
+; CHECK-NEXT: [[PTR2_0_28:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 112
+; CHECK-NEXT: [[TMP369:%.*]] = bitcast i8 [[TMP365]] to b8
+; CHECK-NEXT: [[TMP370:%.*]] = insertelement <4 x b8> poison, b8 [[TMP369]], i32 0
+; CHECK-NEXT: [[TMP371:%.*]] = bitcast i8 [[TMP366]] to b8
+; CHECK-NEXT: [[TMP372:%.*]] = insertelement <4 x b8> [[TMP370]], b8 [[TMP371]], i32 1
+; CHECK-NEXT: [[TMP373:%.*]] = bitcast i8 [[TMP367]] to b8
+; CHECK-NEXT: [[TMP374:%.*]] = insertelement <4 x b8> [[TMP372]], b8 [[TMP373]], i32 2
+; CHECK-NEXT: [[TMP375:%.*]] = bitcast i8 [[TMP368]] to b8
+; CHECK-NEXT: [[TMP376:%.*]] = insertelement <4 x b8> [[TMP374]], b8 [[TMP375]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP376]], ptr [[PTR2_0_28]], align 4
+; CHECK-NEXT: [[PTR_0_29:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 116
+; CHECK-NEXT: [[TMP377:%.*]] = load <4 x b8>, ptr [[PTR_0_29]], align 4
+; CHECK-NEXT: [[LOAD_0_29140:%.*]] = extractelement <4 x b8> [[TMP377]], i32 0
+; CHECK-NEXT: [[TMP378:%.*]] = bitcast b8 [[LOAD_0_29140]] to i8
+; CHECK-NEXT: [[LOAD_1_29141:%.*]] = extractelement <4 x b8> [[TMP377]], i32 1
+; CHECK-NEXT: [[TMP379:%.*]] = bitcast b8 [[LOAD_1_29141]] to i8
+; CHECK-NEXT: [[LOAD_2_29142:%.*]] = extractelement <4 x b8> [[TMP377]], i32 2
+; CHECK-NEXT: [[TMP380:%.*]] = bitcast b8 [[LOAD_2_29142]] to i8
+; CHECK-NEXT: [[LOAD_3_29143:%.*]] = extractelement <4 x b8> [[TMP377]], i32 3
+; CHECK-NEXT: [[TMP381:%.*]] = bitcast b8 [[LOAD_3_29143]] to i8
+; CHECK-NEXT: [[PTR2_0_29:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 116
+; CHECK-NEXT: [[TMP382:%.*]] = bitcast i8 [[TMP378]] to b8
+; CHECK-NEXT: [[TMP383:%.*]] = insertelement <4 x b8> poison, b8 [[TMP382]], i32 0
+; CHECK-NEXT: [[TMP384:%.*]] = bitcast i8 [[TMP379]] to b8
+; CHECK-NEXT: [[TMP385:%.*]] = insertelement <4 x b8> [[TMP383]], b8 [[TMP384]], i32 1
+; CHECK-NEXT: [[TMP386:%.*]] = bitcast i8 [[TMP380]] to b8
+; CHECK-NEXT: [[TMP387:%.*]] = insertelement <4 x b8> [[TMP385]], b8 [[TMP386]], i32 2
+; CHECK-NEXT: [[TMP388:%.*]] = bitcast i8 [[TMP381]] to b8
+; CHECK-NEXT: [[TMP389:%.*]] = insertelement <4 x b8> [[TMP387]], b8 [[TMP388]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP389]], ptr [[PTR2_0_29]], align 4
+; CHECK-NEXT: [[PTR_0_30:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 120
+; CHECK-NEXT: [[TMP390:%.*]] = load <4 x b8>, ptr [[PTR_0_30]], align 4
+; CHECK-NEXT: [[LOAD_0_30144:%.*]] = extractelement <4 x b8> [[TMP390]], i32 0
+; CHECK-NEXT: [[TMP391:%.*]] = bitcast b8 [[LOAD_0_30144]] to i8
+; CHECK-NEXT: [[LOAD_1_30145:%.*]] = extractelement <4 x b8> [[TMP390]], i32 1
+; CHECK-NEXT: [[TMP392:%.*]] = bitcast b8 [[LOAD_1_30145]] to i8
+; CHECK-NEXT: [[LOAD_2_30146:%.*]] = extractelement <4 x b8> [[TMP390]], i32 2
+; CHECK-NEXT: [[TMP393:%.*]] = bitcast b8 [[LOAD_2_30146]] to i8
+; CHECK-NEXT: [[LOAD_3_30147:%.*]] = extractelement <4 x b8> [[TMP390]], i32 3
+; CHECK-NEXT: [[TMP394:%.*]] = bitcast b8 [[LOAD_3_30147]] to i8
+; CHECK-NEXT: [[PTR2_0_30:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 120
+; CHECK-NEXT: [[TMP395:%.*]] = bitcast i8 [[TMP391]] to b8
+; CHECK-NEXT: [[TMP396:%.*]] = insertelement <4 x b8> poison, b8 [[TMP395]], i32 0
+; CHECK-NEXT: [[TMP397:%.*]] = bitcast i8 [[TMP392]] to b8
+; CHECK-NEXT: [[TMP398:%.*]] = insertelement <4 x b8> [[TMP396]], b8 [[TMP397]], i32 1
+; CHECK-NEXT: [[TMP399:%.*]] = bitcast i8 [[TMP393]] to b8
+; CHECK-NEXT: [[TMP400:%.*]] = insertelement <4 x b8> [[TMP398]], b8 [[TMP399]], i32 2
+; CHECK-NEXT: [[TMP401:%.*]] = bitcast i8 [[TMP394]] to b8
+; CHECK-NEXT: [[TMP402:%.*]] = insertelement <4 x b8> [[TMP400]], b8 [[TMP401]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP402]], ptr [[PTR2_0_30]], align 4
+; CHECK-NEXT: [[PTR_0_31:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 124
+; CHECK-NEXT: [[TMP403:%.*]] = load <4 x b8>, ptr [[PTR_0_31]], align 4
+; CHECK-NEXT: [[LOAD_0_31148:%.*]] = extractelement <4 x b8> [[TMP403]], i32 0
+; CHECK-NEXT: [[TMP404:%.*]] = bitcast b8 [[LOAD_0_31148]] to i8
+; CHECK-NEXT: [[LOAD_1_31149:%.*]] = extractelement <4 x b8> [[TMP403]], i32 1
+; CHECK-NEXT: [[TMP405:%.*]] = bitcast b8 [[LOAD_1_31149]] to i8
+; CHECK-NEXT: [[LOAD_2_31150:%.*]] = extractelement <4 x b8> [[TMP403]], i32 2
+; CHECK-NEXT: [[TMP406:%.*]] = bitcast b8 [[LOAD_2_31150]] to i8
+; CHECK-NEXT: [[LOAD_3_31151:%.*]] = extractelement <4 x b8> [[TMP403]], i32 3
+; CHECK-NEXT: [[TMP407:%.*]] = bitcast b8 [[LOAD_3_31151]] to i8
+; CHECK-NEXT: [[PTR2_0_31:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 124
+; CHECK-NEXT: [[TMP408:%.*]] = bitcast i8 [[TMP404]] to b8
+; CHECK-NEXT: [[TMP409:%.*]] = insertelement <4 x b8> poison, b8 [[TMP408]], i32 0
+; CHECK-NEXT: [[TMP410:%.*]] = bitcast i8 [[TMP405]] to b8
+; CHECK-NEXT: [[TMP411:%.*]] = insertelement <4 x b8> [[TMP409]], b8 [[TMP410]], i32 1
+; CHECK-NEXT: [[TMP412:%.*]] = bitcast i8 [[TMP406]] to b8
+; CHECK-NEXT: [[TMP413:%.*]] = insertelement <4 x b8> [[TMP411]], b8 [[TMP412]], i32 2
+; CHECK-NEXT: [[TMP414:%.*]] = bitcast i8 [[TMP407]] to b8
+; CHECK-NEXT: [[TMP415:%.*]] = insertelement <4 x b8> [[TMP413]], b8 [[TMP414]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP415]], ptr [[PTR2_0_31]], align 4
+; CHECK-NEXT: [[PTR_0_32:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 128
+; CHECK-NEXT: [[TMP416:%.*]] = load <4 x b8>, ptr [[PTR_0_32]], align 4
+; CHECK-NEXT: [[LOAD_0_32152:%.*]] = extractelement <4 x b8> [[TMP416]], i32 0
+; CHECK-NEXT: [[TMP417:%.*]] = bitcast b8 [[LOAD_0_32152]] to i8
+; CHECK-NEXT: [[LOAD_1_32153:%.*]] = extractelement <4 x b8> [[TMP416]], i32 1
+; CHECK-NEXT: [[TMP418:%.*]] = bitcast b8 [[LOAD_1_32153]] to i8
+; CHECK-NEXT: [[LOAD_2_32154:%.*]] = extractelement <4 x b8> [[TMP416]], i32 2
+; CHECK-NEXT: [[TMP419:%.*]] = bitcast b8 [[LOAD_2_32154]] to i8
+; CHECK-NEXT: [[LOAD_3_32155:%.*]] = extractelement <4 x b8> [[TMP416]], i32 3
+; CHECK-NEXT: [[TMP420:%.*]] = bitcast b8 [[LOAD_3_32155]] to i8
+; CHECK-NEXT: [[PTR2_0_32:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 128
+; CHECK-NEXT: [[TMP421:%.*]] = bitcast i8 [[TMP417]] to b8
+; CHECK-NEXT: [[TMP422:%.*]] = insertelement <4 x b8> poison, b8 [[TMP421]], i32 0
+; CHECK-NEXT: [[TMP423:%.*]] = bitcast i8 [[TMP418]] to b8
+; CHECK-NEXT: [[TMP424:%.*]] = insertelement <4 x b8> [[TMP422]], b8 [[TMP423]], i32 1
+; CHECK-NEXT: [[TMP425:%.*]] = bitcast i8 [[TMP419]] to b8
+; CHECK-NEXT: [[TMP426:%.*]] = insertelement <4 x b8> [[TMP424]], b8 [[TMP425]], i32 2
+; CHECK-NEXT: [[TMP427:%.*]] = bitcast i8 [[TMP420]] to b8
+; CHECK-NEXT: [[TMP428:%.*]] = insertelement <4 x b8> [[TMP426]], b8 [[TMP427]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP428]], ptr [[PTR2_0_32]], align 4
+; CHECK-NEXT: [[PTR_0_33:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 132
+; CHECK-NEXT: [[TMP429:%.*]] = load <4 x b8>, ptr [[PTR_0_33]], align 4
+; CHECK-NEXT: [[LOAD_0_33156:%.*]] = extractelement <4 x b8> [[TMP429]], i32 0
+; CHECK-NEXT: [[TMP430:%.*]] = bitcast b8 [[LOAD_0_33156]] to i8
+; CHECK-NEXT: [[LOAD_1_33157:%.*]] = extractelement <4 x b8> [[TMP429]], i32 1
+; CHECK-NEXT: [[TMP431:%.*]] = bitcast b8 [[LOAD_1_33157]] to i8
+; CHECK-NEXT: [[LOAD_2_33158:%.*]] = extractelement <4 x b8> [[TMP429]], i32 2
+; CHECK-NEXT: [[TMP432:%.*]] = bitcast b8 [[LOAD_2_33158]] to i8
+; CHECK-NEXT: [[LOAD_3_33159:%.*]] = extractelement <4 x b8> [[TMP429]], i32 3
+; CHECK-NEXT: [[TMP433:%.*]] = bitcast b8 [[LOAD_3_33159]] to i8
+; CHECK-NEXT: [[PTR2_0_33:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 132
+; CHECK-NEXT: [[TMP434:%.*]] = bitcast i8 [[TMP430]] to b8
+; CHECK-NEXT: [[TMP435:%.*]] = insertelement <4 x b8> poison, b8 [[TMP434]], i32 0
+; CHECK-NEXT: [[TMP436:%.*]] = bitcast i8 [[TMP431]] to b8
+; CHECK-NEXT: [[TMP437:%.*]] = insertelement <4 x b8> [[TMP435]], b8 [[TMP436]], i32 1
+; CHECK-NEXT: [[TMP438:%.*]] = bitcast i8 [[TMP432]] to b8
+; CHECK-NEXT: [[TMP439:%.*]] = insertelement <4 x b8> [[TMP437]], b8 [[TMP438]], i32 2
+; CHECK-NEXT: [[TMP440:%.*]] = bitcast i8 [[TMP433]] to b8
+; CHECK-NEXT: [[TMP441:%.*]] = insertelement <4 x b8> [[TMP439]], b8 [[TMP440]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP441]], ptr [[PTR2_0_33]], align 4
+; CHECK-NEXT: [[PTR_0_34:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 136
+; CHECK-NEXT: [[TMP442:%.*]] = load <4 x b8>, ptr [[PTR_0_34]], align 4
+; CHECK-NEXT: [[LOAD_0_34160:%.*]] = extractelement <4 x b8> [[TMP442]], i32 0
+; CHECK-NEXT: [[TMP443:%.*]] = bitcast b8 [[LOAD_0_34160]] to i8
+; CHECK-NEXT: [[LOAD_1_34161:%.*]] = extractelement <4 x b8> [[TMP442]], i32 1
+; CHECK-NEXT: [[TMP444:%.*]] = bitcast b8 [[LOAD_1_34161]] to i8
+; CHECK-NEXT: [[LOAD_2_34162:%.*]] = extractelement <4 x b8> [[TMP442]], i32 2
+; CHECK-NEXT: [[TMP445:%.*]] = bitcast b8 [[LOAD_2_34162]] to i8
+; CHECK-NEXT: [[LOAD_3_34163:%.*]] = extractelement <4 x b8> [[TMP442]], i32 3
+; CHECK-NEXT: [[TMP446:%.*]] = bitcast b8 [[LOAD_3_34163]] to i8
+; CHECK-NEXT: [[PTR2_0_34:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 136
+; CHECK-NEXT: [[TMP447:%.*]] = bitcast i8 [[TMP443]] to b8
+; CHECK-NEXT: [[TMP448:%.*]] = insertelement <4 x b8> poison, b8 [[TMP447]], i32 0
+; CHECK-NEXT: [[TMP449:%.*]] = bitcast i8 [[TMP444]] to b8
+; CHECK-NEXT: [[TMP450:%.*]] = insertelement <4 x b8> [[TMP448]], b8 [[TMP449]], i32 1
+; CHECK-NEXT: [[TMP451:%.*]] = bitcast i8 [[TMP445]] to b8
+; CHECK-NEXT: [[TMP452:%.*]] = insertelement <4 x b8> [[TMP450]], b8 [[TMP451]], i32 2
+; CHECK-NEXT: [[TMP453:%.*]] = bitcast i8 [[TMP446]] to b8
+; CHECK-NEXT: [[TMP454:%.*]] = insertelement <4 x b8> [[TMP452]], b8 [[TMP453]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP454]], ptr [[PTR2_0_34]], align 4
+; CHECK-NEXT: [[PTR_0_35:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 140
+; CHECK-NEXT: [[TMP455:%.*]] = load <4 x b8>, ptr [[PTR_0_35]], align 4
+; CHECK-NEXT: [[LOAD_0_35164:%.*]] = extractelement <4 x b8> [[TMP455]], i32 0
+; CHECK-NEXT: [[TMP456:%.*]] = bitcast b8 [[LOAD_0_35164]] to i8
+; CHECK-NEXT: [[LOAD_1_35165:%.*]] = extractelement <4 x b8> [[TMP455]], i32 1
+; CHECK-NEXT: [[TMP457:%.*]] = bitcast b8 [[LOAD_1_35165]] to i8
+; CHECK-NEXT: [[LOAD_2_35166:%.*]] = extractelement <4 x b8> [[TMP455]], i32 2
+; CHECK-NEXT: [[TMP458:%.*]] = bitcast b8 [[LOAD_2_35166]] to i8
+; CHECK-NEXT: [[LOAD_3_35167:%.*]] = extractelement <4 x b8> [[TMP455]], i32 3
+; CHECK-NEXT: [[TMP459:%.*]] = bitcast b8 [[LOAD_3_35167]] to i8
+; CHECK-NEXT: [[PTR2_0_35:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 140
+; CHECK-NEXT: [[TMP460:%.*]] = bitcast i8 [[TMP456]] to b8
+; CHECK-NEXT: [[TMP461:%.*]] = insertelement <4 x b8> poison, b8 [[TMP460]], i32 0
+; CHECK-NEXT: [[TMP462:%.*]] = bitcast i8 [[TMP457]] to b8
+; CHECK-NEXT: [[TMP463:%.*]] = insertelement <4 x b8> [[TMP461]], b8 [[TMP462]], i32 1
+; CHECK-NEXT: [[TMP464:%.*]] = bitcast i8 [[TMP458]] to b8
+; CHECK-NEXT: [[TMP465:%.*]] = insertelement <4 x b8> [[TMP463]], b8 [[TMP464]], i32 2
+; CHECK-NEXT: [[TMP466:%.*]] = bitcast i8 [[TMP459]] to b8
+; CHECK-NEXT: [[TMP467:%.*]] = insertelement <4 x b8> [[TMP465]], b8 [[TMP466]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP467]], ptr [[PTR2_0_35]], align 4
+; CHECK-NEXT: [[PTR_0_36:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 144
+; CHECK-NEXT: [[TMP468:%.*]] = load <4 x b8>, ptr [[PTR_0_36]], align 4
+; CHECK-NEXT: [[LOAD_0_36168:%.*]] = extractelement <4 x b8> [[TMP468]], i32 0
+; CHECK-NEXT: [[TMP469:%.*]] = bitcast b8 [[LOAD_0_36168]] to i8
+; CHECK-NEXT: [[LOAD_1_36169:%.*]] = extractelement <4 x b8> [[TMP468]], i32 1
+; CHECK-NEXT: [[TMP470:%.*]] = bitcast b8 [[LOAD_1_36169]] to i8
+; CHECK-NEXT: [[LOAD_2_36170:%.*]] = extractelement <4 x b8> [[TMP468]], i32 2
+; CHECK-NEXT: [[TMP471:%.*]] = bitcast b8 [[LOAD_2_36170]] to i8
+; CHECK-NEXT: [[LOAD_3_36171:%.*]] = extractelement <4 x b8> [[TMP468]], i32 3
+; CHECK-NEXT: [[TMP472:%.*]] = bitcast b8 [[LOAD_3_36171]] to i8
+; CHECK-NEXT: [[PTR2_0_36:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 144
+; CHECK-NEXT: [[TMP473:%.*]] = bitcast i8 [[TMP469]] to b8
+; CHECK-NEXT: [[TMP474:%.*]] = insertelement <4 x b8> poison, b8 [[TMP473]], i32 0
+; CHECK-NEXT: [[TMP475:%.*]] = bitcast i8 [[TMP470]] to b8
+; CHECK-NEXT: [[TMP476:%.*]] = insertelement <4 x b8> [[TMP474]], b8 [[TMP475]], i32 1
+; CHECK-NEXT: [[TMP477:%.*]] = bitcast i8 [[TMP471]] to b8
+; CHECK-NEXT: [[TMP478:%.*]] = insertelement <4 x b8> [[TMP476]], b8 [[TMP477]], i32 2
+; CHECK-NEXT: [[TMP479:%.*]] = bitcast i8 [[TMP472]] to b8
+; CHECK-NEXT: [[TMP480:%.*]] = insertelement <4 x b8> [[TMP478]], b8 [[TMP479]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP480]], ptr [[PTR2_0_36]], align 4
+; CHECK-NEXT: [[PTR_0_37:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 148
+; CHECK-NEXT: [[TMP481:%.*]] = load <4 x b8>, ptr [[PTR_0_37]], align 4
+; CHECK-NEXT: [[LOAD_0_37172:%.*]] = extractelement <4 x b8> [[TMP481]], i32 0
+; CHECK-NEXT: [[TMP482:%.*]] = bitcast b8 [[LOAD_0_37172]] to i8
+; CHECK-NEXT: [[LOAD_1_37173:%.*]] = extractelement <4 x b8> [[TMP481]], i32 1
+; CHECK-NEXT: [[TMP483:%.*]] = bitcast b8 [[LOAD_1_37173]] to i8
+; CHECK-NEXT: [[LOAD_2_37174:%.*]] = extractelement <4 x b8> [[TMP481]], i32 2
+; CHECK-NEXT: [[TMP484:%.*]] = bitcast b8 [[LOAD_2_37174]] to i8
+; CHECK-NEXT: [[LOAD_3_37175:%.*]] = extractelement <4 x b8> [[TMP481]], i32 3
+; CHECK-NEXT: [[TMP485:%.*]] = bitcast b8 [[LOAD_3_37175]] to i8
+; CHECK-NEXT: [[PTR2_0_37:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 148
+; CHECK-NEXT: [[TMP486:%.*]] = bitcast i8 [[TMP482]] to b8
+; CHECK-NEXT: [[TMP487:%.*]] = insertelement <4 x b8> poison, b8 [[TMP486]], i32 0
+; CHECK-NEXT: [[TMP488:%.*]] = bitcast i8 [[TMP483]] to b8
+; CHECK-NEXT: [[TMP489:%.*]] = insertelement <4 x b8> [[TMP487]], b8 [[TMP488]], i32 1
+; CHECK-NEXT: [[TMP490:%.*]] = bitcast i8 [[TMP484]] to b8
+; CHECK-NEXT: [[TMP491:%.*]] = insertelement <4 x b8> [[TMP489]], b8 [[TMP490]], i32 2
+; CHECK-NEXT: [[TMP492:%.*]] = bitcast i8 [[TMP485]] to b8
+; CHECK-NEXT: [[TMP493:%.*]] = insertelement <4 x b8> [[TMP491]], b8 [[TMP492]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP493]], ptr [[PTR2_0_37]], align 4
+; CHECK-NEXT: [[PTR_0_38:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 152
+; CHECK-NEXT: [[TMP494:%.*]] = load <4 x b8>, ptr [[PTR_0_38]], align 4
+; CHECK-NEXT: [[LOAD_0_38176:%.*]] = extractelement <4 x b8> [[TMP494]], i32 0
+; CHECK-NEXT: [[TMP495:%.*]] = bitcast b8 [[LOAD_0_38176]] to i8
+; CHECK-NEXT: [[LOAD_1_38177:%.*]] = extractelement <4 x b8> [[TMP494]], i32 1
+; CHECK-NEXT: [[TMP496:%.*]] = bitcast b8 [[LOAD_1_38177]] to i8
+; CHECK-NEXT: [[LOAD_2_38178:%.*]] = extractelement <4 x b8> [[TMP494]], i32 2
+; CHECK-NEXT: [[TMP497:%.*]] = bitcast b8 [[LOAD_2_38178]] to i8
+; CHECK-NEXT: [[LOAD_3_38179:%.*]] = extractelement <4 x b8> [[TMP494]], i32 3
+; CHECK-NEXT: [[TMP498:%.*]] = bitcast b8 [[LOAD_3_38179]] to i8
+; CHECK-NEXT: [[PTR2_0_38:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 152
+; CHECK-NEXT: [[TMP499:%.*]] = bitcast i8 [[TMP495]] to b8
+; CHECK-NEXT: [[TMP500:%.*]] = insertelement <4 x b8> poison, b8 [[TMP499]], i32 0
+; CHECK-NEXT: [[TMP501:%.*]] = bitcast i8 [[TMP496]] to b8
+; CHECK-NEXT: [[TMP502:%.*]] = insertelement <4 x b8> [[TMP500]], b8 [[TMP501]], i32 1
+; CHECK-NEXT: [[TMP503:%.*]] = bitcast i8 [[TMP497]] to b8
+; CHECK-NEXT: [[TMP504:%.*]] = insertelement <4 x b8> [[TMP502]], b8 [[TMP503]], i32 2
+; CHECK-NEXT: [[TMP505:%.*]] = bitcast i8 [[TMP498]] to b8
+; CHECK-NEXT: [[TMP506:%.*]] = insertelement <4 x b8> [[TMP504]], b8 [[TMP505]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP506]], ptr [[PTR2_0_38]], align 4
+; CHECK-NEXT: [[PTR_0_39:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 156
+; CHECK-NEXT: [[TMP507:%.*]] = load <4 x b8>, ptr [[PTR_0_39]], align 4
+; CHECK-NEXT: [[LOAD_0_39180:%.*]] = extractelement <4 x b8> [[TMP507]], i32 0
+; CHECK-NEXT: [[TMP508:%.*]] = bitcast b8 [[LOAD_0_39180]] to i8
+; CHECK-NEXT: [[LOAD_1_39181:%.*]] = extractelement <4 x b8> [[TMP507]], i32 1
+; CHECK-NEXT: [[TMP509:%.*]] = bitcast b8 [[LOAD_1_39181]] to i8
+; CHECK-NEXT: [[LOAD_2_39182:%.*]] = extractelement <4 x b8> [[TMP507]], i32 2
+; CHECK-NEXT: [[TMP510:%.*]] = bitcast b8 [[LOAD_2_39182]] to i8
+; CHECK-NEXT: [[LOAD_3_39183:%.*]] = extractelement <4 x b8> [[TMP507]], i32 3
+; CHECK-NEXT: [[TMP511:%.*]] = bitcast b8 [[LOAD_3_39183]] to i8
+; CHECK-NEXT: [[PTR2_0_39:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 156
+; CHECK-NEXT: [[TMP512:%.*]] = bitcast i8 [[TMP508]] to b8
+; CHECK-NEXT: [[TMP513:%.*]] = insertelement <4 x b8> poison, b8 [[TMP512]], i32 0
+; CHECK-NEXT: [[TMP514:%.*]] = bitcast i8 [[TMP509]] to b8
+; CHECK-NEXT: [[TMP515:%.*]] = insertelement <4 x b8> [[TMP513]], b8 [[TMP514]], i32 1
+; CHECK-NEXT: [[TMP516:%.*]] = bitcast i8 [[TMP510]] to b8
+; CHECK-NEXT: [[TMP517:%.*]] = insertelement <4 x b8> [[TMP515]], b8 [[TMP516]], i32 2
+; CHECK-NEXT: [[TMP518:%.*]] = bitcast i8 [[TMP511]] to b8
+; CHECK-NEXT: [[TMP519:%.*]] = insertelement <4 x b8> [[TMP517]], b8 [[TMP518]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP519]], ptr [[PTR2_0_39]], align 4
+; CHECK-NEXT: [[PTR_0_40:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 160
+; CHECK-NEXT: [[TMP520:%.*]] = load <4 x b8>, ptr [[PTR_0_40]], align 4
+; CHECK-NEXT: [[LOAD_0_40184:%.*]] = extractelement <4 x b8> [[TMP520]], i32 0
+; CHECK-NEXT: [[TMP521:%.*]] = bitcast b8 [[LOAD_0_40184]] to i8
+; CHECK-NEXT: [[LOAD_1_40185:%.*]] = extractelement <4 x b8> [[TMP520]], i32 1
+; CHECK-NEXT: [[TMP522:%.*]] = bitcast b8 [[LOAD_1_40185]] to i8
+; CHECK-NEXT: [[LOAD_2_40186:%.*]] = extractelement <4 x b8> [[TMP520]], i32 2
+; CHECK-NEXT: [[TMP523:%.*]] = bitcast b8 [[LOAD_2_40186]] to i8
+; CHECK-NEXT: [[LOAD_3_40187:%.*]] = extractelement <4 x b8> [[TMP520]], i32 3
+; CHECK-NEXT: [[TMP524:%.*]] = bitcast b8 [[LOAD_3_40187]] to i8
+; CHECK-NEXT: [[PTR2_0_40:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 160
+; CHECK-NEXT: [[TMP525:%.*]] = bitcast i8 [[TMP521]] to b8
+; CHECK-NEXT: [[TMP526:%.*]] = insertelement <4 x b8> poison, b8 [[TMP525]], i32 0
+; CHECK-NEXT: [[TMP527:%.*]] = bitcast i8 [[TMP522]] to b8
+; CHECK-NEXT: [[TMP528:%.*]] = insertelement <4 x b8> [[TMP526]], b8 [[TMP527]], i32 1
+; CHECK-NEXT: [[TMP529:%.*]] = bitcast i8 [[TMP523]] to b8
+; CHECK-NEXT: [[TMP530:%.*]] = insertelement <4 x b8> [[TMP528]], b8 [[TMP529]], i32 2
+; CHECK-NEXT: [[TMP531:%.*]] = bitcast i8 [[TMP524]] to b8
+; CHECK-NEXT: [[TMP532:%.*]] = insertelement <4 x b8> [[TMP530]], b8 [[TMP531]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP532]], ptr [[PTR2_0_40]], align 4
+; CHECK-NEXT: [[PTR_0_41:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 164
+; CHECK-NEXT: [[TMP533:%.*]] = load <4 x b8>, ptr [[PTR_0_41]], align 4
+; CHECK-NEXT: [[LOAD_0_41188:%.*]] = extractelement <4 x b8> [[TMP533]], i32 0
+; CHECK-NEXT: [[TMP534:%.*]] = bitcast b8 [[LOAD_0_41188]] to i8
+; CHECK-NEXT: [[LOAD_1_41189:%.*]] = extractelement <4 x b8> [[TMP533]], i32 1
+; CHECK-NEXT: [[TMP535:%.*]] = bitcast b8 [[LOAD_1_41189]] to i8
+; CHECK-NEXT: [[LOAD_2_41190:%.*]] = extractelement <4 x b8> [[TMP533]], i32 2
+; CHECK-NEXT: [[TMP536:%.*]] = bitcast b8 [[LOAD_2_41190]] to i8
+; CHECK-NEXT: [[LOAD_3_41191:%.*]] = extractelement <4 x b8> [[TMP533]], i32 3
+; CHECK-NEXT: [[TMP537:%.*]] = bitcast b8 [[LOAD_3_41191]] to i8
+; CHECK-NEXT: [[PTR2_0_41:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 164
+; CHECK-NEXT: [[TMP538:%.*]] = bitcast i8 [[TMP534]] to b8
+; CHECK-NEXT: [[TMP539:%.*]] = insertelement <4 x b8> poison, b8 [[TMP538]], i32 0
+; CHECK-NEXT: [[TMP540:%.*]] = bitcast i8 [[TMP535]] to b8
+; CHECK-NEXT: [[TMP541:%.*]] = insertelement <4 x b8> [[TMP539]], b8 [[TMP540]], i32 1
+; CHECK-NEXT: [[TMP542:%.*]] = bitcast i8 [[TMP536]] to b8
+; CHECK-NEXT: [[TMP543:%.*]] = insertelement <4 x b8> [[TMP541]], b8 [[TMP542]], i32 2
+; CHECK-NEXT: [[TMP544:%.*]] = bitcast i8 [[TMP537]] to b8
+; CHECK-NEXT: [[TMP545:%.*]] = insertelement <4 x b8> [[TMP543]], b8 [[TMP544]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP545]], ptr [[PTR2_0_41]], align 4
+; CHECK-NEXT: [[PTR_0_42:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 168
+; CHECK-NEXT: [[TMP546:%.*]] = load <4 x b8>, ptr [[PTR_0_42]], align 4
+; CHECK-NEXT: [[LOAD_0_42192:%.*]] = extractelement <4 x b8> [[TMP546]], i32 0
+; CHECK-NEXT: [[TMP547:%.*]] = bitcast b8 [[LOAD_0_42192]] to i8
+; CHECK-NEXT: [[LOAD_1_42193:%.*]] = extractelement <4 x b8> [[TMP546]], i32 1
+; CHECK-NEXT: [[TMP548:%.*]] = bitcast b8 [[LOAD_1_42193]] to i8
+; CHECK-NEXT: [[LOAD_2_42194:%.*]] = extractelement <4 x b8> [[TMP546]], i32 2
+; CHECK-NEXT: [[TMP549:%.*]] = bitcast b8 [[LOAD_2_42194]] to i8
+; CHECK-NEXT: [[LOAD_3_42195:%.*]] = extractelement <4 x b8> [[TMP546]], i32 3
+; CHECK-NEXT: [[TMP550:%.*]] = bitcast b8 [[LOAD_3_42195]] to i8
+; CHECK-NEXT: [[PTR2_0_42:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 168
+; CHECK-NEXT: [[TMP551:%.*]] = bitcast i8 [[TMP547]] to b8
+; CHECK-NEXT: [[TMP552:%.*]] = insertelement <4 x b8> poison, b8 [[TMP551]], i32 0
+; CHECK-NEXT: [[TMP553:%.*]] = bitcast i8 [[TMP548]] to b8
+; CHECK-NEXT: [[TMP554:%.*]] = insertelement <4 x b8> [[TMP552]], b8 [[TMP553]], i32 1
+; CHECK-NEXT: [[TMP555:%.*]] = bitcast i8 [[TMP549]] to b8
+; CHECK-NEXT: [[TMP556:%.*]] = insertelement <4 x b8> [[TMP554]], b8 [[TMP555]], i32 2
+; CHECK-NEXT: [[TMP557:%.*]] = bitcast i8 [[TMP550]] to b8
+; CHECK-NEXT: [[TMP558:%.*]] = insertelement <4 x b8> [[TMP556]], b8 [[TMP557]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP558]], ptr [[PTR2_0_42]], align 4
+; CHECK-NEXT: [[PTR_0_43:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 172
+; CHECK-NEXT: [[TMP559:%.*]] = load <4 x b8>, ptr [[PTR_0_43]], align 4
+; CHECK-NEXT: [[LOAD_0_43196:%.*]] = extractelement <4 x b8> [[TMP559]], i32 0
+; CHECK-NEXT: [[TMP560:%.*]] = bitcast b8 [[LOAD_0_43196]] to i8
+; CHECK-NEXT: [[LOAD_1_43197:%.*]] = extractelement <4 x b8> [[TMP559]], i32 1
+; CHECK-NEXT: [[TMP561:%.*]] = bitcast b8 [[LOAD_1_43197]] to i8
+; CHECK-NEXT: [[LOAD_2_43198:%.*]] = extractelement <4 x b8> [[TMP559]], i32 2
+; CHECK-NEXT: [[TMP562:%.*]] = bitcast b8 [[LOAD_2_43198]] to i8
+; CHECK-NEXT: [[LOAD_3_43199:%.*]] = extractelement <4 x b8> [[TMP559]], i32 3
+; CHECK-NEXT: [[TMP563:%.*]] = bitcast b8 [[LOAD_3_43199]] to i8
+; CHECK-NEXT: [[PTR2_0_43:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 172
+; CHECK-NEXT: [[TMP564:%.*]] = bitcast i8 [[TMP560]] to b8
+; CHECK-NEXT: [[TMP565:%.*]] = insertelement <4 x b8> poison, b8 [[TMP564]], i32 0
+; CHECK-NEXT: [[TMP566:%.*]] = bitcast i8 [[TMP561]] to b8
+; CHECK-NEXT: [[TMP567:%.*]] = insertelement <4 x b8> [[TMP565]], b8 [[TMP566]], i32 1
+; CHECK-NEXT: [[TMP568:%.*]] = bitcast i8 [[TMP562]] to b8
+; CHECK-NEXT: [[TMP569:%.*]] = insertelement <4 x b8> [[TMP567]], b8 [[TMP568]], i32 2
+; CHECK-NEXT: [[TMP570:%.*]] = bitcast i8 [[TMP563]] to b8
+; CHECK-NEXT: [[TMP571:%.*]] = insertelement <4 x b8> [[TMP569]], b8 [[TMP570]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP571]], ptr [[PTR2_0_43]], align 4
+; CHECK-NEXT: [[PTR_0_44:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 176
+; CHECK-NEXT: [[TMP572:%.*]] = load <4 x b8>, ptr [[PTR_0_44]], align 4
+; CHECK-NEXT: [[LOAD_0_44200:%.*]] = extractelement <4 x b8> [[TMP572]], i32 0
+; CHECK-NEXT: [[TMP573:%.*]] = bitcast b8 [[LOAD_0_44200]] to i8
+; CHECK-NEXT: [[LOAD_1_44201:%.*]] = extractelement <4 x b8> [[TMP572]], i32 1
+; CHECK-NEXT: [[TMP574:%.*]] = bitcast b8 [[LOAD_1_44201]] to i8
+; CHECK-NEXT: [[LOAD_2_44202:%.*]] = extractelement <4 x b8> [[TMP572]], i32 2
+; CHECK-NEXT: [[TMP575:%.*]] = bitcast b8 [[LOAD_2_44202]] to i8
+; CHECK-NEXT: [[LOAD_3_44203:%.*]] = extractelement <4 x b8> [[TMP572]], i32 3
+; CHECK-NEXT: [[TMP576:%.*]] = bitcast b8 [[LOAD_3_44203]] to i8
+; CHECK-NEXT: [[PTR2_0_44:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 176
+; CHECK-NEXT: [[TMP577:%.*]] = bitcast i8 [[TMP573]] to b8
+; CHECK-NEXT: [[TMP578:%.*]] = insertelement <4 x b8> poison, b8 [[TMP577]], i32 0
+; CHECK-NEXT: [[TMP579:%.*]] = bitcast i8 [[TMP574]] to b8
+; CHECK-NEXT: [[TMP580:%.*]] = insertelement <4 x b8> [[TMP578]], b8 [[TMP579]], i32 1
+; CHECK-NEXT: [[TMP581:%.*]] = bitcast i8 [[TMP575]] to b8
+; CHECK-NEXT: [[TMP582:%.*]] = insertelement <4 x b8> [[TMP580]], b8 [[TMP581]], i32 2
+; CHECK-NEXT: [[TMP583:%.*]] = bitcast i8 [[TMP576]] to b8
+; CHECK-NEXT: [[TMP584:%.*]] = insertelement <4 x b8> [[TMP582]], b8 [[TMP583]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP584]], ptr [[PTR2_0_44]], align 4
+; CHECK-NEXT: [[PTR_0_45:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 180
+; CHECK-NEXT: [[TMP585:%.*]] = load <4 x b8>, ptr [[PTR_0_45]], align 4
+; CHECK-NEXT: [[LOAD_0_45204:%.*]] = extractelement <4 x b8> [[TMP585]], i32 0
+; CHECK-NEXT: [[TMP586:%.*]] = bitcast b8 [[LOAD_0_45204]] to i8
+; CHECK-NEXT: [[LOAD_1_45205:%.*]] = extractelement <4 x b8> [[TMP585]], i32 1
+; CHECK-NEXT: [[TMP587:%.*]] = bitcast b8 [[LOAD_1_45205]] to i8
+; CHECK-NEXT: [[LOAD_2_45206:%.*]] = extractelement <4 x b8> [[TMP585]], i32 2
+; CHECK-NEXT: [[TMP588:%.*]] = bitcast b8 [[LOAD_2_45206]] to i8
+; CHECK-NEXT: [[LOAD_3_45207:%.*]] = extractelement <4 x b8> [[TMP585]], i32 3
+; CHECK-NEXT: [[TMP589:%.*]] = bitcast b8 [[LOAD_3_45207]] to i8
+; CHECK-NEXT: [[PTR2_0_45:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 180
+; CHECK-NEXT: [[TMP590:%.*]] = bitcast i8 [[TMP586]] to b8
+; CHECK-NEXT: [[TMP591:%.*]] = insertelement <4 x b8> poison, b8 [[TMP590]], i32 0
+; CHECK-NEXT: [[TMP592:%.*]] = bitcast i8 [[TMP587]] to b8
+; CHECK-NEXT: [[TMP593:%.*]] = insertelement <4 x b8> [[TMP591]], b8 [[TMP592]], i32 1
+; CHECK-NEXT: [[TMP594:%.*]] = bitcast i8 [[TMP588]] to b8
+; CHECK-NEXT: [[TMP595:%.*]] = insertelement <4 x b8> [[TMP593]], b8 [[TMP594]], i32 2
+; CHECK-NEXT: [[TMP596:%.*]] = bitcast i8 [[TMP589]] to b8
+; CHECK-NEXT: [[TMP597:%.*]] = insertelement <4 x b8> [[TMP595]], b8 [[TMP596]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP597]], ptr [[PTR2_0_45]], align 4
+; CHECK-NEXT: [[PTR_0_46:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 184
+; CHECK-NEXT: [[TMP598:%.*]] = load <4 x b8>, ptr [[PTR_0_46]], align 4
+; CHECK-NEXT: [[LOAD_0_46208:%.*]] = extractelement <4 x b8> [[TMP598]], i32 0
+; CHECK-NEXT: [[TMP599:%.*]] = bitcast b8 [[LOAD_0_46208]] to i8
+; CHECK-NEXT: [[LOAD_1_46209:%.*]] = extractelement <4 x b8> [[TMP598]], i32 1
+; CHECK-NEXT: [[TMP600:%.*]] = bitcast b8 [[LOAD_1_46209]] to i8
+; CHECK-NEXT: [[LOAD_2_46210:%.*]] = extractelement <4 x b8> [[TMP598]], i32 2
+; CHECK-NEXT: [[TMP601:%.*]] = bitcast b8 [[LOAD_2_46210]] to i8
+; CHECK-NEXT: [[LOAD_3_46211:%.*]] = extractelement <4 x b8> [[TMP598]], i32 3
+; CHECK-NEXT: [[TMP602:%.*]] = bitcast b8 [[LOAD_3_46211]] to i8
+; CHECK-NEXT: [[PTR2_0_46:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 184
+; CHECK-NEXT: [[TMP603:%.*]] = bitcast i8 [[TMP599]] to b8
+; CHECK-NEXT: [[TMP604:%.*]] = insertelement <4 x b8> poison, b8 [[TMP603]], i32 0
+; CHECK-NEXT: [[TMP605:%.*]] = bitcast i8 [[TMP600]] to b8
+; CHECK-NEXT: [[TMP606:%.*]] = insertelement <4 x b8> [[TMP604]], b8 [[TMP605]], i32 1
+; CHECK-NEXT: [[TMP607:%.*]] = bitcast i8 [[TMP601]] to b8
+; CHECK-NEXT: [[TMP608:%.*]] = insertelement <4 x b8> [[TMP606]], b8 [[TMP607]], i32 2
+; CHECK-NEXT: [[TMP609:%.*]] = bitcast i8 [[TMP602]] to b8
+; CHECK-NEXT: [[TMP610:%.*]] = insertelement <4 x b8> [[TMP608]], b8 [[TMP609]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP610]], ptr [[PTR2_0_46]], align 4
+; CHECK-NEXT: [[PTR_0_47:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 188
+; CHECK-NEXT: [[TMP611:%.*]] = load <4 x b8>, ptr [[PTR_0_47]], align 4
+; CHECK-NEXT: [[LOAD_0_47212:%.*]] = extractelement <4 x b8> [[TMP611]], i32 0
+; CHECK-NEXT: [[TMP612:%.*]] = bitcast b8 [[LOAD_0_47212]] to i8
+; CHECK-NEXT: [[LOAD_1_47213:%.*]] = extractelement <4 x b8> [[TMP611]], i32 1
+; CHECK-NEXT: [[TMP613:%.*]] = bitcast b8 [[LOAD_1_47213]] to i8
+; CHECK-NEXT: [[LOAD_2_47214:%.*]] = extractelement <4 x b8> [[TMP611]], i32 2
+; CHECK-NEXT: [[TMP614:%.*]] = bitcast b8 [[LOAD_2_47214]] to i8
+; CHECK-NEXT: [[LOAD_3_47215:%.*]] = extractelement <4 x b8> [[TMP611]], i32 3
+; CHECK-NEXT: [[TMP615:%.*]] = bitcast b8 [[LOAD_3_47215]] to i8
+; CHECK-NEXT: [[PTR2_0_47:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 188
+; CHECK-NEXT: [[TMP616:%.*]] = bitcast i8 [[TMP612]] to b8
+; CHECK-NEXT: [[TMP617:%.*]] = insertelement <4 x b8> poison, b8 [[TMP616]], i32 0
+; CHECK-NEXT: [[TMP618:%.*]] = bitcast i8 [[TMP613]] to b8
+; CHECK-NEXT: [[TMP619:%.*]] = insertelement <4 x b8> [[TMP617]], b8 [[TMP618]], i32 1
+; CHECK-NEXT: [[TMP620:%.*]] = bitcast i8 [[TMP614]] to b8
+; CHECK-NEXT: [[TMP621:%.*]] = insertelement <4 x b8> [[TMP619]], b8 [[TMP620]], i32 2
+; CHECK-NEXT: [[TMP622:%.*]] = bitcast i8 [[TMP615]] to b8
+; CHECK-NEXT: [[TMP623:%.*]] = insertelement <4 x b8> [[TMP621]], b8 [[TMP622]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP623]], ptr [[PTR2_0_47]], align 4
+; CHECK-NEXT: [[PTR_0_48:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 192
+; CHECK-NEXT: [[TMP624:%.*]] = load <4 x b8>, ptr [[PTR_0_48]], align 4
+; CHECK-NEXT: [[LOAD_0_48216:%.*]] = extractelement <4 x b8> [[TMP624]], i32 0
+; CHECK-NEXT: [[TMP625:%.*]] = bitcast b8 [[LOAD_0_48216]] to i8
+; CHECK-NEXT: [[LOAD_1_48217:%.*]] = extractelement <4 x b8> [[TMP624]], i32 1
+; CHECK-NEXT: [[TMP626:%.*]] = bitcast b8 [[LOAD_1_48217]] to i8
+; CHECK-NEXT: [[LOAD_2_48218:%.*]] = extractelement <4 x b8> [[TMP624]], i32 2
+; CHECK-NEXT: [[TMP627:%.*]] = bitcast b8 [[LOAD_2_48218]] to i8
+; CHECK-NEXT: [[LOAD_3_48219:%.*]] = extractelement <4 x b8> [[TMP624]], i32 3
+; CHECK-NEXT: [[TMP628:%.*]] = bitcast b8 [[LOAD_3_48219]] to i8
+; CHECK-NEXT: [[PTR2_0_48:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 192
+; CHECK-NEXT: [[TMP629:%.*]] = bitcast i8 [[TMP625]] to b8
+; CHECK-NEXT: [[TMP630:%.*]] = insertelement <4 x b8> poison, b8 [[TMP629]], i32 0
+; CHECK-NEXT: [[TMP631:%.*]] = bitcast i8 [[TMP626]] to b8
+; CHECK-NEXT: [[TMP632:%.*]] = insertelement <4 x b8> [[TMP630]], b8 [[TMP631]], i32 1
+; CHECK-NEXT: [[TMP633:%.*]] = bitcast i8 [[TMP627]] to b8
+; CHECK-NEXT: [[TMP634:%.*]] = insertelement <4 x b8> [[TMP632]], b8 [[TMP633]], i32 2
+; CHECK-NEXT: [[TMP635:%.*]] = bitcast i8 [[TMP628]] to b8
+; CHECK-NEXT: [[TMP636:%.*]] = insertelement <4 x b8> [[TMP634]], b8 [[TMP635]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP636]], ptr [[PTR2_0_48]], align 4
+; CHECK-NEXT: [[PTR_0_49:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 196
+; CHECK-NEXT: [[TMP637:%.*]] = load <4 x b8>, ptr [[PTR_0_49]], align 4
+; CHECK-NEXT: [[LOAD_0_49220:%.*]] = extractelement <4 x b8> [[TMP637]], i32 0
+; CHECK-NEXT: [[TMP638:%.*]] = bitcast b8 [[LOAD_0_49220]] to i8
+; CHECK-NEXT: [[LOAD_1_49221:%.*]] = extractelement <4 x b8> [[TMP637]], i32 1
+; CHECK-NEXT: [[TMP639:%.*]] = bitcast b8 [[LOAD_1_49221]] to i8
+; CHECK-NEXT: [[LOAD_2_49222:%.*]] = extractelement <4 x b8> [[TMP637]], i32 2
+; CHECK-NEXT: [[TMP640:%.*]] = bitcast b8 [[LOAD_2_49222]] to i8
+; CHECK-NEXT: [[LOAD_3_49223:%.*]] = extractelement <4 x b8> [[TMP637]], i32 3
+; CHECK-NEXT: [[TMP641:%.*]] = bitcast b8 [[LOAD_3_49223]] to i8
+; CHECK-NEXT: [[PTR2_0_49:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 196
+; CHECK-NEXT: [[TMP642:%.*]] = bitcast i8 [[TMP638]] to b8
+; CHECK-NEXT: [[TMP643:%.*]] = insertelement <4 x b8> poison, b8 [[TMP642]], i32 0
+; CHECK-NEXT: [[TMP644:%.*]] = bitcast i8 [[TMP639]] to b8
+; CHECK-NEXT: [[TMP645:%.*]] = insertelement <4 x b8> [[TMP643]], b8 [[TMP644]], i32 1
+; CHECK-NEXT: [[TMP646:%.*]] = bitcast i8 [[TMP640]] to b8
+; CHECK-NEXT: [[TMP647:%.*]] = insertelement <4 x b8> [[TMP645]], b8 [[TMP646]], i32 2
+; CHECK-NEXT: [[TMP648:%.*]] = bitcast i8 [[TMP641]] to b8
+; CHECK-NEXT: [[TMP649:%.*]] = insertelement <4 x b8> [[TMP647]], b8 [[TMP648]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP649]], ptr [[PTR2_0_49]], align 4
+; CHECK-NEXT: [[PTR_0_50:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 200
+; CHECK-NEXT: [[TMP650:%.*]] = load <4 x b8>, ptr [[PTR_0_50]], align 4
+; CHECK-NEXT: [[LOAD_0_50224:%.*]] = extractelement <4 x b8> [[TMP650]], i32 0
+; CHECK-NEXT: [[TMP651:%.*]] = bitcast b8 [[LOAD_0_50224]] to i8
+; CHECK-NEXT: [[LOAD_1_50225:%.*]] = extractelement <4 x b8> [[TMP650]], i32 1
+; CHECK-NEXT: [[TMP652:%.*]] = bitcast b8 [[LOAD_1_50225]] to i8
+; CHECK-NEXT: [[LOAD_2_50226:%.*]] = extractelement <4 x b8> [[TMP650]], i32 2
+; CHECK-NEXT: [[TMP653:%.*]] = bitcast b8 [[LOAD_2_50226]] to i8
+; CHECK-NEXT: [[LOAD_3_50227:%.*]] = extractelement <4 x b8> [[TMP650]], i32 3
+; CHECK-NEXT: [[TMP654:%.*]] = bitcast b8 [[LOAD_3_50227]] to i8
+; CHECK-NEXT: [[PTR2_0_50:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 200
+; CHECK-NEXT: [[TMP655:%.*]] = bitcast i8 [[TMP651]] to b8
+; CHECK-NEXT: [[TMP656:%.*]] = insertelement <4 x b8> poison, b8 [[TMP655]], i32 0
+; CHECK-NEXT: [[TMP657:%.*]] = bitcast i8 [[TMP652]] to b8
+; CHECK-NEXT: [[TMP658:%.*]] = insertelement <4 x b8> [[TMP656]], b8 [[TMP657]], i32 1
+; CHECK-NEXT: [[TMP659:%.*]] = bitcast i8 [[TMP653]] to b8
+; CHECK-NEXT: [[TMP660:%.*]] = insertelement <4 x b8> [[TMP658]], b8 [[TMP659]], i32 2
+; CHECK-NEXT: [[TMP661:%.*]] = bitcast i8 [[TMP654]] to b8
+; CHECK-NEXT: [[TMP662:%.*]] = insertelement <4 x b8> [[TMP660]], b8 [[TMP661]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP662]], ptr [[PTR2_0_50]], align 4
+; CHECK-NEXT: [[PTR_0_51:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 204
+; CHECK-NEXT: [[TMP663:%.*]] = load <4 x b8>, ptr [[PTR_0_51]], align 4
+; CHECK-NEXT: [[LOAD_0_51228:%.*]] = extractelement <4 x b8> [[TMP663]], i32 0
+; CHECK-NEXT: [[TMP664:%.*]] = bitcast b8 [[LOAD_0_51228]] to i8
+; CHECK-NEXT: [[LOAD_1_51229:%.*]] = extractelement <4 x b8> [[TMP663]], i32 1
+; CHECK-NEXT: [[TMP665:%.*]] = bitcast b8 [[LOAD_1_51229]] to i8
+; CHECK-NEXT: [[LOAD_2_51230:%.*]] = extractelement <4 x b8> [[TMP663]], i32 2
+; CHECK-NEXT: [[TMP666:%.*]] = bitcast b8 [[LOAD_2_51230]] to i8
+; CHECK-NEXT: [[LOAD_3_51231:%.*]] = extractelement <4 x b8> [[TMP663]], i32 3
+; CHECK-NEXT: [[TMP667:%.*]] = bitcast b8 [[LOAD_3_51231]] to i8
+; CHECK-NEXT: [[PTR2_0_51:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 204
+; CHECK-NEXT: [[TMP668:%.*]] = bitcast i8 [[TMP664]] to b8
+; CHECK-NEXT: [[TMP669:%.*]] = insertelement <4 x b8> poison, b8 [[TMP668]], i32 0
+; CHECK-NEXT: [[TMP670:%.*]] = bitcast i8 [[TMP665]] to b8
+; CHECK-NEXT: [[TMP671:%.*]] = insertelement <4 x b8> [[TMP669]], b8 [[TMP670]], i32 1
+; CHECK-NEXT: [[TMP672:%.*]] = bitcast i8 [[TMP666]] to b8
+; CHECK-NEXT: [[TMP673:%.*]] = insertelement <4 x b8> [[TMP671]], b8 [[TMP672]], i32 2
+; CHECK-NEXT: [[TMP674:%.*]] = bitcast i8 [[TMP667]] to b8
+; CHECK-NEXT: [[TMP675:%.*]] = insertelement <4 x b8> [[TMP673]], b8 [[TMP674]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP675]], ptr [[PTR2_0_51]], align 4
+; CHECK-NEXT: [[PTR_0_52:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 208
+; CHECK-NEXT: [[TMP676:%.*]] = load <4 x b8>, ptr [[PTR_0_52]], align 4
+; CHECK-NEXT: [[LOAD_0_52232:%.*]] = extractelement <4 x b8> [[TMP676]], i32 0
+; CHECK-NEXT: [[TMP677:%.*]] = bitcast b8 [[LOAD_0_52232]] to i8
+; CHECK-NEXT: [[LOAD_1_52233:%.*]] = extractelement <4 x b8> [[TMP676]], i32 1
+; CHECK-NEXT: [[TMP678:%.*]] = bitcast b8 [[LOAD_1_52233]] to i8
+; CHECK-NEXT: [[LOAD_2_52234:%.*]] = extractelement <4 x b8> [[TMP676]], i32 2
+; CHECK-NEXT: [[TMP679:%.*]] = bitcast b8 [[LOAD_2_52234]] to i8
+; CHECK-NEXT: [[LOAD_3_52235:%.*]] = extractelement <4 x b8> [[TMP676]], i32 3
+; CHECK-NEXT: [[TMP680:%.*]] = bitcast b8 [[LOAD_3_52235]] to i8
+; CHECK-NEXT: [[PTR2_0_52:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 208
+; CHECK-NEXT: [[TMP681:%.*]] = bitcast i8 [[TMP677]] to b8
+; CHECK-NEXT: [[TMP682:%.*]] = insertelement <4 x b8> poison, b8 [[TMP681]], i32 0
+; CHECK-NEXT: [[TMP683:%.*]] = bitcast i8 [[TMP678]] to b8
+; CHECK-NEXT: [[TMP684:%.*]] = insertelement <4 x b8> [[TMP682]], b8 [[TMP683]], i32 1
+; CHECK-NEXT: [[TMP685:%.*]] = bitcast i8 [[TMP679]] to b8
+; CHECK-NEXT: [[TMP686:%.*]] = insertelement <4 x b8> [[TMP684]], b8 [[TMP685]], i32 2
+; CHECK-NEXT: [[TMP687:%.*]] = bitcast i8 [[TMP680]] to b8
+; CHECK-NEXT: [[TMP688:%.*]] = insertelement <4 x b8> [[TMP686]], b8 [[TMP687]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP688]], ptr [[PTR2_0_52]], align 4
+; CHECK-NEXT: [[PTR_0_53:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 212
+; CHECK-NEXT: [[TMP689:%.*]] = load <4 x b8>, ptr [[PTR_0_53]], align 4
+; CHECK-NEXT: [[LOAD_0_53236:%.*]] = extractelement <4 x b8> [[TMP689]], i32 0
+; CHECK-NEXT: [[TMP690:%.*]] = bitcast b8 [[LOAD_0_53236]] to i8
+; CHECK-NEXT: [[LOAD_1_53237:%.*]] = extractelement <4 x b8> [[TMP689]], i32 1
+; CHECK-NEXT: [[TMP691:%.*]] = bitcast b8 [[LOAD_1_53237]] to i8
+; CHECK-NEXT: [[LOAD_2_53238:%.*]] = extractelement <4 x b8> [[TMP689]], i32 2
+; CHECK-NEXT: [[TMP692:%.*]] = bitcast b8 [[LOAD_2_53238]] to i8
+; CHECK-NEXT: [[LOAD_3_53239:%.*]] = extractelement <4 x b8> [[TMP689]], i32 3
+; CHECK-NEXT: [[TMP693:%.*]] = bitcast b8 [[LOAD_3_53239]] to i8
+; CHECK-NEXT: [[PTR2_0_53:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 212
+; CHECK-NEXT: [[TMP694:%.*]] = bitcast i8 [[TMP690]] to b8
+; CHECK-NEXT: [[TMP695:%.*]] = insertelement <4 x b8> poison, b8 [[TMP694]], i32 0
+; CHECK-NEXT: [[TMP696:%.*]] = bitcast i8 [[TMP691]] to b8
+; CHECK-NEXT: [[TMP697:%.*]] = insertelement <4 x b8> [[TMP695]], b8 [[TMP696]], i32 1
+; CHECK-NEXT: [[TMP698:%.*]] = bitcast i8 [[TMP692]] to b8
+; CHECK-NEXT: [[TMP699:%.*]] = insertelement <4 x b8> [[TMP697]], b8 [[TMP698]], i32 2
+; CHECK-NEXT: [[TMP700:%.*]] = bitcast i8 [[TMP693]] to b8
+; CHECK-NEXT: [[TMP701:%.*]] = insertelement <4 x b8> [[TMP699]], b8 [[TMP700]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP701]], ptr [[PTR2_0_53]], align 4
+; CHECK-NEXT: [[PTR_0_54:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 216
+; CHECK-NEXT: [[TMP702:%.*]] = load <4 x b8>, ptr [[PTR_0_54]], align 4
+; CHECK-NEXT: [[LOAD_0_54240:%.*]] = extractelement <4 x b8> [[TMP702]], i32 0
+; CHECK-NEXT: [[TMP703:%.*]] = bitcast b8 [[LOAD_0_54240]] to i8
+; CHECK-NEXT: [[LOAD_1_54241:%.*]] = extractelement <4 x b8> [[TMP702]], i32 1
+; CHECK-NEXT: [[TMP704:%.*]] = bitcast b8 [[LOAD_1_54241]] to i8
+; CHECK-NEXT: [[LOAD_2_54242:%.*]] = extractelement <4 x b8> [[TMP702]], i32 2
+; CHECK-NEXT: [[TMP705:%.*]] = bitcast b8 [[LOAD_2_54242]] to i8
+; CHECK-NEXT: [[LOAD_3_54243:%.*]] = extractelement <4 x b8> [[TMP702]], i32 3
+; CHECK-NEXT: [[TMP706:%.*]] = bitcast b8 [[LOAD_3_54243]] to i8
+; CHECK-NEXT: [[PTR2_0_54:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 216
+; CHECK-NEXT: [[TMP707:%.*]] = bitcast i8 [[TMP703]] to b8
+; CHECK-NEXT: [[TMP708:%.*]] = insertelement <4 x b8> poison, b8 [[TMP707]], i32 0
+; CHECK-NEXT: [[TMP709:%.*]] = bitcast i8 [[TMP704]] to b8
+; CHECK-NEXT: [[TMP710:%.*]] = insertelement <4 x b8> [[TMP708]], b8 [[TMP709]], i32 1
+; CHECK-NEXT: [[TMP711:%.*]] = bitcast i8 [[TMP705]] to b8
+; CHECK-NEXT: [[TMP712:%.*]] = insertelement <4 x b8> [[TMP710]], b8 [[TMP711]], i32 2
+; CHECK-NEXT: [[TMP713:%.*]] = bitcast i8 [[TMP706]] to b8
+; CHECK-NEXT: [[TMP714:%.*]] = insertelement <4 x b8> [[TMP712]], b8 [[TMP713]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP714]], ptr [[PTR2_0_54]], align 4
+; CHECK-NEXT: [[PTR_0_55:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 220
+; CHECK-NEXT: [[TMP715:%.*]] = load <4 x b8>, ptr [[PTR_0_55]], align 4
+; CHECK-NEXT: [[LOAD_0_55244:%.*]] = extractelement <4 x b8> [[TMP715]], i32 0
+; CHECK-NEXT: [[TMP716:%.*]] = bitcast b8 [[LOAD_0_55244]] to i8
+; CHECK-NEXT: [[LOAD_1_55245:%.*]] = extractelement <4 x b8> [[TMP715]], i32 1
+; CHECK-NEXT: [[TMP717:%.*]] = bitcast b8 [[LOAD_1_55245]] to i8
+; CHECK-NEXT: [[LOAD_2_55246:%.*]] = extractelement <4 x b8> [[TMP715]], i32 2
+; CHECK-NEXT: [[TMP718:%.*]] = bitcast b8 [[LOAD_2_55246]] to i8
+; CHECK-NEXT: [[LOAD_3_55247:%.*]] = extractelement <4 x b8> [[TMP715]], i32 3
+; CHECK-NEXT: [[TMP719:%.*]] = bitcast b8 [[LOAD_3_55247]] to i8
+; CHECK-NEXT: [[PTR2_0_55:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 220
+; CHECK-NEXT: [[TMP720:%.*]] = bitcast i8 [[TMP716]] to b8
+; CHECK-NEXT: [[TMP721:%.*]] = insertelement <4 x b8> poison, b8 [[TMP720]], i32 0
+; CHECK-NEXT: [[TMP722:%.*]] = bitcast i8 [[TMP717]] to b8
+; CHECK-NEXT: [[TMP723:%.*]] = insertelement <4 x b8> [[TMP721]], b8 [[TMP722]], i32 1
+; CHECK-NEXT: [[TMP724:%.*]] = bitcast i8 [[TMP718]] to b8
+; CHECK-NEXT: [[TMP725:%.*]] = insertelement <4 x b8> [[TMP723]], b8 [[TMP724]], i32 2
+; CHECK-NEXT: [[TMP726:%.*]] = bitcast i8 [[TMP719]] to b8
+; CHECK-NEXT: [[TMP727:%.*]] = insertelement <4 x b8> [[TMP725]], b8 [[TMP726]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP727]], ptr [[PTR2_0_55]], align 4
+; CHECK-NEXT: [[PTR_0_56:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 224
+; CHECK-NEXT: [[TMP728:%.*]] = load <4 x b8>, ptr [[PTR_0_56]], align 4
+; CHECK-NEXT: [[LOAD_0_56248:%.*]] = extractelement <4 x b8> [[TMP728]], i32 0
+; CHECK-NEXT: [[TMP729:%.*]] = bitcast b8 [[LOAD_0_56248]] to i8
+; CHECK-NEXT: [[LOAD_1_56249:%.*]] = extractelement <4 x b8> [[TMP728]], i32 1
+; CHECK-NEXT: [[TMP730:%.*]] = bitcast b8 [[LOAD_1_56249]] to i8
+; CHECK-NEXT: [[LOAD_2_56250:%.*]] = extractelement <4 x b8> [[TMP728]], i32 2
+; CHECK-NEXT: [[TMP731:%.*]] = bitcast b8 [[LOAD_2_56250]] to i8
+; CHECK-NEXT: [[LOAD_3_56251:%.*]] = extractelement <4 x b8> [[TMP728]], i32 3
+; CHECK-NEXT: [[TMP732:%.*]] = bitcast b8 [[LOAD_3_56251]] to i8
+; CHECK-NEXT: [[PTR2_0_56:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 224
+; CHECK-NEXT: [[TMP733:%.*]] = bitcast i8 [[TMP729]] to b8
+; CHECK-NEXT: [[TMP734:%.*]] = insertelement <4 x b8> poison, b8 [[TMP733]], i32 0
+; CHECK-NEXT: [[TMP735:%.*]] = bitcast i8 [[TMP730]] to b8
+; CHECK-NEXT: [[TMP736:%.*]] = insertelement <4 x b8> [[TMP734]], b8 [[TMP735]], i32 1
+; CHECK-NEXT: [[TMP737:%.*]] = bitcast i8 [[TMP731]] to b8
+; CHECK-NEXT: [[TMP738:%.*]] = insertelement <4 x b8> [[TMP736]], b8 [[TMP737]], i32 2
+; CHECK-NEXT: [[TMP739:%.*]] = bitcast i8 [[TMP732]] to b8
+; CHECK-NEXT: [[TMP740:%.*]] = insertelement <4 x b8> [[TMP738]], b8 [[TMP739]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP740]], ptr [[PTR2_0_56]], align 4
+; CHECK-NEXT: [[PTR_0_57:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 228
+; CHECK-NEXT: [[TMP741:%.*]] = load <4 x b8>, ptr [[PTR_0_57]], align 4
+; CHECK-NEXT: [[LOAD_0_57252:%.*]] = extractelement <4 x b8> [[TMP741]], i32 0
+; CHECK-NEXT: [[TMP742:%.*]] = bitcast b8 [[LOAD_0_57252]] to i8
+; CHECK-NEXT: [[LOAD_1_57253:%.*]] = extractelement <4 x b8> [[TMP741]], i32 1
+; CHECK-NEXT: [[TMP743:%.*]] = bitcast b8 [[LOAD_1_57253]] to i8
+; CHECK-NEXT: [[LOAD_2_57254:%.*]] = extractelement <4 x b8> [[TMP741]], i32 2
+; CHECK-NEXT: [[TMP744:%.*]] = bitcast b8 [[LOAD_2_57254]] to i8
+; CHECK-NEXT: [[LOAD_3_57255:%.*]] = extractelement <4 x b8> [[TMP741]], i32 3
+; CHECK-NEXT: [[TMP745:%.*]] = bitcast b8 [[LOAD_3_57255]] to i8
+; CHECK-NEXT: [[PTR2_0_57:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 228
+; CHECK-NEXT: [[TMP746:%.*]] = bitcast i8 [[TMP742]] to b8
+; CHECK-NEXT: [[TMP747:%.*]] = insertelement <4 x b8> poison, b8 [[TMP746]], i32 0
+; CHECK-NEXT: [[TMP748:%.*]] = bitcast i8 [[TMP743]] to b8
+; CHECK-NEXT: [[TMP749:%.*]] = insertelement <4 x b8> [[TMP747]], b8 [[TMP748]], i32 1
+; CHECK-NEXT: [[TMP750:%.*]] = bitcast i8 [[TMP744]] to b8
+; CHECK-NEXT: [[TMP751:%.*]] = insertelement <4 x b8> [[TMP749]], b8 [[TMP750]], i32 2
+; CHECK-NEXT: [[TMP752:%.*]] = bitcast i8 [[TMP745]] to b8
+; CHECK-NEXT: [[TMP753:%.*]] = insertelement <4 x b8> [[TMP751]], b8 [[TMP752]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP753]], ptr [[PTR2_0_57]], align 4
+; CHECK-NEXT: [[PTR_0_58:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 232
+; CHECK-NEXT: [[TMP754:%.*]] = load <4 x b8>, ptr [[PTR_0_58]], align 4
+; CHECK-NEXT: [[LOAD_0_58256:%.*]] = extractelement <4 x b8> [[TMP754]], i32 0
+; CHECK-NEXT: [[TMP755:%.*]] = bitcast b8 [[LOAD_0_58256]] to i8
+; CHECK-NEXT: [[LOAD_1_58257:%.*]] = extractelement <4 x b8> [[TMP754]], i32 1
+; CHECK-NEXT: [[TMP756:%.*]] = bitcast b8 [[LOAD_1_58257]] to i8
+; CHECK-NEXT: [[LOAD_2_58258:%.*]] = extractelement <4 x b8> [[TMP754]], i32 2
+; CHECK-NEXT: [[TMP757:%.*]] = bitcast b8 [[LOAD_2_58258]] to i8
+; CHECK-NEXT: [[LOAD_3_58259:%.*]] = extractelement <4 x b8> [[TMP754]], i32 3
+; CHECK-NEXT: [[TMP758:%.*]] = bitcast b8 [[LOAD_3_58259]] to i8
+; CHECK-NEXT: [[PTR2_0_58:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 232
+; CHECK-NEXT: [[TMP759:%.*]] = bitcast i8 [[TMP755]] to b8
+; CHECK-NEXT: [[TMP760:%.*]] = insertelement <4 x b8> poison, b8 [[TMP759]], i32 0
+; CHECK-NEXT: [[TMP761:%.*]] = bitcast i8 [[TMP756]] to b8
+; CHECK-NEXT: [[TMP762:%.*]] = insertelement <4 x b8> [[TMP760]], b8 [[TMP761]], i32 1
+; CHECK-NEXT: [[TMP763:%.*]] = bitcast i8 [[TMP757]] to b8
+; CHECK-NEXT: [[TMP764:%.*]] = insertelement <4 x b8> [[TMP762]], b8 [[TMP763]], i32 2
+; CHECK-NEXT: [[TMP765:%.*]] = bitcast i8 [[TMP758]] to b8
+; CHECK-NEXT: [[TMP766:%.*]] = insertelement <4 x b8> [[TMP764]], b8 [[TMP765]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP766]], ptr [[PTR2_0_58]], align 4
+; CHECK-NEXT: [[PTR_0_59:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 236
+; CHECK-NEXT: [[TMP767:%.*]] = load <4 x b8>, ptr [[PTR_0_59]], align 4
+; CHECK-NEXT: [[LOAD_0_59260:%.*]] = extractelement <4 x b8> [[TMP767]], i32 0
+; CHECK-NEXT: [[TMP768:%.*]] = bitcast b8 [[LOAD_0_59260]] to i8
+; CHECK-NEXT: [[LOAD_1_59261:%.*]] = extractelement <4 x b8> [[TMP767]], i32 1
+; CHECK-NEXT: [[TMP769:%.*]] = bitcast b8 [[LOAD_1_59261]] to i8
+; CHECK-NEXT: [[LOAD_2_59262:%.*]] = extractelement <4 x b8> [[TMP767]], i32 2
+; CHECK-NEXT: [[TMP770:%.*]] = bitcast b8 [[LOAD_2_59262]] to i8
+; CHECK-NEXT: [[LOAD_3_59263:%.*]] = extractelement <4 x b8> [[TMP767]], i32 3
+; CHECK-NEXT: [[TMP771:%.*]] = bitcast b8 [[LOAD_3_59263]] to i8
+; CHECK-NEXT: [[PTR2_0_59:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 236
+; CHECK-NEXT: [[TMP772:%.*]] = bitcast i8 [[TMP768]] to b8
+; CHECK-NEXT: [[TMP773:%.*]] = insertelement <4 x b8> poison, b8 [[TMP772]], i32 0
+; CHECK-NEXT: [[TMP774:%.*]] = bitcast i8 [[TMP769]] to b8
+; CHECK-NEXT: [[TMP775:%.*]] = insertelement <4 x b8> [[TMP773]], b8 [[TMP774]], i32 1
+; CHECK-NEXT: [[TMP776:%.*]] = bitcast i8 [[TMP770]] to b8
+; CHECK-NEXT: [[TMP777:%.*]] = insertelement <4 x b8> [[TMP775]], b8 [[TMP776]], i32 2
+; CHECK-NEXT: [[TMP778:%.*]] = bitcast i8 [[TMP771]] to b8
+; CHECK-NEXT: [[TMP779:%.*]] = insertelement <4 x b8> [[TMP777]], b8 [[TMP778]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP779]], ptr [[PTR2_0_59]], align 4
+; CHECK-NEXT: [[PTR_0_60:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 240
+; CHECK-NEXT: [[TMP780:%.*]] = load <4 x b8>, ptr [[PTR_0_60]], align 4
+; CHECK-NEXT: [[LOAD_0_60264:%.*]] = extractelement <4 x b8> [[TMP780]], i32 0
+; CHECK-NEXT: [[TMP781:%.*]] = bitcast b8 [[LOAD_0_60264]] to i8
+; CHECK-NEXT: [[LOAD_1_60265:%.*]] = extractelement <4 x b8> [[TMP780]], i32 1
+; CHECK-NEXT: [[TMP782:%.*]] = bitcast b8 [[LOAD_1_60265]] to i8
+; CHECK-NEXT: [[LOAD_2_60266:%.*]] = extractelement <4 x b8> [[TMP780]], i32 2
+; CHECK-NEXT: [[TMP783:%.*]] = bitcast b8 [[LOAD_2_60266]] to i8
+; CHECK-NEXT: [[LOAD_3_60267:%.*]] = extractelement <4 x b8> [[TMP780]], i32 3
+; CHECK-NEXT: [[TMP784:%.*]] = bitcast b8 [[LOAD_3_60267]] to i8
+; CHECK-NEXT: [[PTR2_0_60:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 240
+; CHECK-NEXT: [[TMP785:%.*]] = bitcast i8 [[TMP781]] to b8
+; CHECK-NEXT: [[TMP786:%.*]] = insertelement <4 x b8> poison, b8 [[TMP785]], i32 0
+; CHECK-NEXT: [[TMP787:%.*]] = bitcast i8 [[TMP782]] to b8
+; CHECK-NEXT: [[TMP788:%.*]] = insertelement <4 x b8> [[TMP786]], b8 [[TMP787]], i32 1
+; CHECK-NEXT: [[TMP789:%.*]] = bitcast i8 [[TMP783]] to b8
+; CHECK-NEXT: [[TMP790:%.*]] = insertelement <4 x b8> [[TMP788]], b8 [[TMP789]], i32 2
+; CHECK-NEXT: [[TMP791:%.*]] = bitcast i8 [[TMP784]] to b8
+; CHECK-NEXT: [[TMP792:%.*]] = insertelement <4 x b8> [[TMP790]], b8 [[TMP791]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP792]], ptr [[PTR2_0_60]], align 4
+; CHECK-NEXT: [[PTR_0_61:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 244
+; CHECK-NEXT: [[TMP793:%.*]] = load <4 x b8>, ptr [[PTR_0_61]], align 4
+; CHECK-NEXT: [[LOAD_0_61268:%.*]] = extractelement <4 x b8> [[TMP793]], i32 0
+; CHECK-NEXT: [[TMP794:%.*]] = bitcast b8 [[LOAD_0_61268]] to i8
+; CHECK-NEXT: [[LOAD_1_61269:%.*]] = extractelement <4 x b8> [[TMP793]], i32 1
+; CHECK-NEXT: [[TMP795:%.*]] = bitcast b8 [[LOAD_1_61269]] to i8
+; CHECK-NEXT: [[LOAD_2_61270:%.*]] = extractelement <4 x b8> [[TMP793]], i32 2
+; CHECK-NEXT: [[TMP796:%.*]] = bitcast b8 [[LOAD_2_61270]] to i8
+; CHECK-NEXT: [[LOAD_3_61271:%.*]] = extractelement <4 x b8> [[TMP793]], i32 3
+; CHECK-NEXT: [[TMP797:%.*]] = bitcast b8 [[LOAD_3_61271]] to i8
+; CHECK-NEXT: [[PTR2_0_61:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 244
+; CHECK-NEXT: [[TMP798:%.*]] = bitcast i8 [[TMP794]] to b8
+; CHECK-NEXT: [[TMP799:%.*]] = insertelement <4 x b8> poison, b8 [[TMP798]], i32 0
+; CHECK-NEXT: [[TMP800:%.*]] = bitcast i8 [[TMP795]] to b8
+; CHECK-NEXT: [[TMP801:%.*]] = insertelement <4 x b8> [[TMP799]], b8 [[TMP800]], i32 1
+; CHECK-NEXT: [[TMP802:%.*]] = bitcast i8 [[TMP796]] to b8
+; CHECK-NEXT: [[TMP803:%.*]] = insertelement <4 x b8> [[TMP801]], b8 [[TMP802]], i32 2
+; CHECK-NEXT: [[TMP804:%.*]] = bitcast i8 [[TMP797]] to b8
+; CHECK-NEXT: [[TMP805:%.*]] = insertelement <4 x b8> [[TMP803]], b8 [[TMP804]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP805]], ptr [[PTR2_0_61]], align 4
+; CHECK-NEXT: [[PTR_0_62:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 248
+; CHECK-NEXT: [[TMP806:%.*]] = load <4 x b8>, ptr [[PTR_0_62]], align 4
+; CHECK-NEXT: [[LOAD_0_62272:%.*]] = extractelement <4 x b8> [[TMP806]], i32 0
+; CHECK-NEXT: [[TMP807:%.*]] = bitcast b8 [[LOAD_0_62272]] to i8
+; CHECK-NEXT: [[LOAD_1_62273:%.*]] = extractelement <4 x b8> [[TMP806]], i32 1
+; CHECK-NEXT: [[TMP808:%.*]] = bitcast b8 [[LOAD_1_62273]] to i8
+; CHECK-NEXT: [[LOAD_2_62274:%.*]] = extractelement <4 x b8> [[TMP806]], i32 2
+; CHECK-NEXT: [[TMP809:%.*]] = bitcast b8 [[LOAD_2_62274]] to i8
+; CHECK-NEXT: [[LOAD_3_62275:%.*]] = extractelement <4 x b8> [[TMP806]], i32 3
+; CHECK-NEXT: [[TMP810:%.*]] = bitcast b8 [[LOAD_3_62275]] to i8
+; CHECK-NEXT: [[PTR2_0_62:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 248
+; CHECK-NEXT: [[TMP811:%.*]] = bitcast i8 [[TMP807]] to b8
+; CHECK-NEXT: [[TMP812:%.*]] = insertelement <4 x b8> poison, b8 [[TMP811]], i32 0
+; CHECK-NEXT: [[TMP813:%.*]] = bitcast i8 [[TMP808]] to b8
+; CHECK-NEXT: [[TMP814:%.*]] = insertelement <4 x b8> [[TMP812]], b8 [[TMP813]], i32 1
+; CHECK-NEXT: [[TMP815:%.*]] = bitcast i8 [[TMP809]] to b8
+; CHECK-NEXT: [[TMP816:%.*]] = insertelement <4 x b8> [[TMP814]], b8 [[TMP815]], i32 2
+; CHECK-NEXT: [[TMP817:%.*]] = bitcast i8 [[TMP810]] to b8
+; CHECK-NEXT: [[TMP818:%.*]] = insertelement <4 x b8> [[TMP816]], b8 [[TMP817]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP818]], ptr [[PTR2_0_62]], align 4
+; CHECK-NEXT: [[PTR_0_63:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 252
+; CHECK-NEXT: [[TMP819:%.*]] = load <4 x b8>, ptr [[PTR_0_63]], align 4
+; CHECK-NEXT: [[LOAD_0_63276:%.*]] = extractelement <4 x b8> [[TMP819]], i32 0
+; CHECK-NEXT: [[TMP820:%.*]] = bitcast b8 [[LOAD_0_63276]] to i8
+; CHECK-NEXT: [[LOAD_1_63277:%.*]] = extractelement <4 x b8> [[TMP819]], i32 1
+; CHECK-NEXT: [[TMP821:%.*]] = bitcast b8 [[LOAD_1_63277]] to i8
+; CHECK-NEXT: [[LOAD_2_63278:%.*]] = extractelement <4 x b8> [[TMP819]], i32 2
+; CHECK-NEXT: [[TMP822:%.*]] = bitcast b8 [[LOAD_2_63278]] to i8
+; CHECK-NEXT: [[LOAD_3_63279:%.*]] = extractelement <4 x b8> [[TMP819]], i32 3
+; CHECK-NEXT: [[TMP823:%.*]] = bitcast b8 [[LOAD_3_63279]] to i8
+; CHECK-NEXT: [[PTR2_0_63:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 252
+; CHECK-NEXT: [[TMP824:%.*]] = bitcast i8 [[TMP820]] to b8
+; CHECK-NEXT: [[TMP825:%.*]] = insertelement <4 x b8> poison, b8 [[TMP824]], i32 0
+; CHECK-NEXT: [[TMP826:%.*]] = bitcast i8 [[TMP821]] to b8
+; CHECK-NEXT: [[TMP827:%.*]] = insertelement <4 x b8> [[TMP825]], b8 [[TMP826]], i32 1
+; CHECK-NEXT: [[TMP828:%.*]] = bitcast i8 [[TMP822]] to b8
+; CHECK-NEXT: [[TMP829:%.*]] = insertelement <4 x b8> [[TMP827]], b8 [[TMP828]], i32 2
+; CHECK-NEXT: [[TMP830:%.*]] = bitcast i8 [[TMP823]] to b8
+; CHECK-NEXT: [[TMP831:%.*]] = insertelement <4 x b8> [[TMP829]], b8 [[TMP830]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP831]], ptr [[PTR2_0_63]], align 4
+; CHECK-NEXT: [[PTR_0_64:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 256
+; CHECK-NEXT: [[TMP832:%.*]] = load <4 x b8>, ptr [[PTR_0_64]], align 4
+; CHECK-NEXT: [[LOAD_0_64280:%.*]] = extractelement <4 x b8> [[TMP832]], i32 0
+; CHECK-NEXT: [[TMP833:%.*]] = bitcast b8 [[LOAD_0_64280]] to i8
+; CHECK-NEXT: [[LOAD_1_64281:%.*]] = extractelement <4 x b8> [[TMP832]], i32 1
+; CHECK-NEXT: [[TMP834:%.*]] = bitcast b8 [[LOAD_1_64281]] to i8
+; CHECK-NEXT: [[LOAD_2_64282:%.*]] = extractelement <4 x b8> [[TMP832]], i32 2
+; CHECK-NEXT: [[TMP835:%.*]] = bitcast b8 [[LOAD_2_64282]] to i8
+; CHECK-NEXT: [[LOAD_3_64283:%.*]] = extractelement <4 x b8> [[TMP832]], i32 3
+; CHECK-NEXT: [[TMP836:%.*]] = bitcast b8 [[LOAD_3_64283]] to i8
+; CHECK-NEXT: [[PTR2_0_64:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 256
+; CHECK-NEXT: [[TMP837:%.*]] = bitcast i8 [[TMP833]] to b8
+; CHECK-NEXT: [[TMP838:%.*]] = insertelement <4 x b8> poison, b8 [[TMP837]], i32 0
+; CHECK-NEXT: [[TMP839:%.*]] = bitcast i8 [[TMP834]] to b8
+; CHECK-NEXT: [[TMP840:%.*]] = insertelement <4 x b8> [[TMP838]], b8 [[TMP839]], i32 1
+; CHECK-NEXT: [[TMP841:%.*]] = bitcast i8 [[TMP835]] to b8
+; CHECK-NEXT: [[TMP842:%.*]] = insertelement <4 x b8> [[TMP840]], b8 [[TMP841]], i32 2
+; CHECK-NEXT: [[TMP843:%.*]] = bitcast i8 [[TMP836]] to b8
+; CHECK-NEXT: [[TMP844:%.*]] = insertelement <4 x b8> [[TMP842]], b8 [[TMP843]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP844]], ptr [[PTR2_0_64]], align 4
+; CHECK-NEXT: [[PTR_0_65:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 260
+; CHECK-NEXT: [[TMP845:%.*]] = load <4 x b8>, ptr [[PTR_0_65]], align 4
+; CHECK-NEXT: [[LOAD_0_65284:%.*]] = extractelement <4 x b8> [[TMP845]], i32 0
+; CHECK-NEXT: [[TMP846:%.*]] = bitcast b8 [[LOAD_0_65284]] to i8
+; CHECK-NEXT: [[LOAD_1_65285:%.*]] = extractelement <4 x b8> [[TMP845]], i32 1
+; CHECK-NEXT: [[TMP847:%.*]] = bitcast b8 [[LOAD_1_65285]] to i8
+; CHECK-NEXT: [[LOAD_2_65286:%.*]] = extractelement <4 x b8> [[TMP845]], i32 2
+; CHECK-NEXT: [[TMP848:%.*]] = bitcast b8 [[LOAD_2_65286]] to i8
+; CHECK-NEXT: [[LOAD_3_65287:%.*]] = extractelement <4 x b8> [[TMP845]], i32 3
+; CHECK-NEXT: [[TMP849:%.*]] = bitcast b8 [[LOAD_3_65287]] to i8
+; CHECK-NEXT: [[PTR2_0_65:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 260
+; CHECK-NEXT: [[TMP850:%.*]] = bitcast i8 [[TMP846]] to b8
+; CHECK-NEXT: [[TMP851:%.*]] = insertelement <4 x b8> poison, b8 [[TMP850]], i32 0
+; CHECK-NEXT: [[TMP852:%.*]] = bitcast i8 [[TMP847]] to b8
+; CHECK-NEXT: [[TMP853:%.*]] = insertelement <4 x b8> [[TMP851]], b8 [[TMP852]], i32 1
+; CHECK-NEXT: [[TMP854:%.*]] = bitcast i8 [[TMP848]] to b8
+; CHECK-NEXT: [[TMP855:%.*]] = insertelement <4 x b8> [[TMP853]], b8 [[TMP854]], i32 2
+; CHECK-NEXT: [[TMP856:%.*]] = bitcast i8 [[TMP849]] to b8
+; CHECK-NEXT: [[TMP857:%.*]] = insertelement <4 x b8> [[TMP855]], b8 [[TMP856]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP857]], ptr [[PTR2_0_65]], align 4
+; CHECK-NEXT: [[PTR_0_66:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 264
+; CHECK-NEXT: [[TMP858:%.*]] = load <4 x b8>, ptr [[PTR_0_66]], align 4
+; CHECK-NEXT: [[LOAD_0_66288:%.*]] = extractelement <4 x b8> [[TMP858]], i32 0
+; CHECK-NEXT: [[TMP859:%.*]] = bitcast b8 [[LOAD_0_66288]] to i8
+; CHECK-NEXT: [[LOAD_1_66289:%.*]] = extractelement <4 x b8> [[TMP858]], i32 1
+; CHECK-NEXT: [[TMP860:%.*]] = bitcast b8 [[LOAD_1_66289]] to i8
+; CHECK-NEXT: [[LOAD_2_66290:%.*]] = extractelement <4 x b8> [[TMP858]], i32 2
+; CHECK-NEXT: [[TMP861:%.*]] = bitcast b8 [[LOAD_2_66290]] to i8
+; CHECK-NEXT: [[LOAD_3_66291:%.*]] = extractelement <4 x b8> [[TMP858]], i32 3
+; CHECK-NEXT: [[TMP862:%.*]] = bitcast b8 [[LOAD_3_66291]] to i8
+; CHECK-NEXT: [[PTR2_0_66:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 264
+; CHECK-NEXT: [[TMP863:%.*]] = bitcast i8 [[TMP859]] to b8
+; CHECK-NEXT: [[TMP864:%.*]] = insertelement <4 x b8> poison, b8 [[TMP863]], i32 0
+; CHECK-NEXT: [[TMP865:%.*]] = bitcast i8 [[TMP860]] to b8
+; CHECK-NEXT: [[TMP866:%.*]] = insertelement <4 x b8> [[TMP864]], b8 [[TMP865]], i32 1
+; CHECK-NEXT: [[TMP867:%.*]] = bitcast i8 [[TMP861]] to b8
+; CHECK-NEXT: [[TMP868:%.*]] = insertelement <4 x b8> [[TMP866]], b8 [[TMP867]], i32 2
+; CHECK-NEXT: [[TMP869:%.*]] = bitcast i8 [[TMP862]] to b8
+; CHECK-NEXT: [[TMP870:%.*]] = insertelement <4 x b8> [[TMP868]], b8 [[TMP869]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP870]], ptr [[PTR2_0_66]], align 4
+; CHECK-NEXT: [[PTR_0_67:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 268
+; CHECK-NEXT: [[TMP871:%.*]] = load <4 x b8>, ptr [[PTR_0_67]], align 4
+; CHECK-NEXT: [[LOAD_0_67292:%.*]] = extractelement <4 x b8> [[TMP871]], i32 0
+; CHECK-NEXT: [[TMP872:%.*]] = bitcast b8 [[LOAD_0_67292]] to i8
+; CHECK-NEXT: [[LOAD_1_67293:%.*]] = extractelement <4 x b8> [[TMP871]], i32 1
+; CHECK-NEXT: [[TMP873:%.*]] = bitcast b8 [[LOAD_1_67293]] to i8
+; CHECK-NEXT: [[LOAD_2_67294:%.*]] = extractelement <4 x b8> [[TMP871]], i32 2
+; CHECK-NEXT: [[TMP874:%.*]] = bitcast b8 [[LOAD_2_67294]] to i8
+; CHECK-NEXT: [[LOAD_3_67295:%.*]] = extractelement <4 x b8> [[TMP871]], i32 3
+; CHECK-NEXT: [[TMP875:%.*]] = bitcast b8 [[LOAD_3_67295]] to i8
+; CHECK-NEXT: [[PTR2_0_67:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 268
+; CHECK-NEXT: [[TMP876:%.*]] = bitcast i8 [[TMP872]] to b8
+; CHECK-NEXT: [[TMP877:%.*]] = insertelement <4 x b8> poison, b8 [[TMP876]], i32 0
+; CHECK-NEXT: [[TMP878:%.*]] = bitcast i8 [[TMP873]] to b8
+; CHECK-NEXT: [[TMP879:%.*]] = insertelement <4 x b8> [[TMP877]], b8 [[TMP878]], i32 1
+; CHECK-NEXT: [[TMP880:%.*]] = bitcast i8 [[TMP874]] to b8
+; CHECK-NEXT: [[TMP881:%.*]] = insertelement <4 x b8> [[TMP879]], b8 [[TMP880]], i32 2
+; CHECK-NEXT: [[TMP882:%.*]] = bitcast i8 [[TMP875]] to b8
+; CHECK-NEXT: [[TMP883:%.*]] = insertelement <4 x b8> [[TMP881]], b8 [[TMP882]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP883]], ptr [[PTR2_0_67]], align 4
+; CHECK-NEXT: [[PTR_0_68:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 272
+; CHECK-NEXT: [[TMP884:%.*]] = load <4 x b8>, ptr [[PTR_0_68]], align 4
+; CHECK-NEXT: [[LOAD_0_68296:%.*]] = extractelement <4 x b8> [[TMP884]], i32 0
+; CHECK-NEXT: [[TMP885:%.*]] = bitcast b8 [[LOAD_0_68296]] to i8
+; CHECK-NEXT: [[LOAD_1_68297:%.*]] = extractelement <4 x b8> [[TMP884]], i32 1
+; CHECK-NEXT: [[TMP886:%.*]] = bitcast b8 [[LOAD_1_68297]] to i8
+; CHECK-NEXT: [[LOAD_2_68298:%.*]] = extractelement <4 x b8> [[TMP884]], i32 2
+; CHECK-NEXT: [[TMP887:%.*]] = bitcast b8 [[LOAD_2_68298]] to i8
+; CHECK-NEXT: [[LOAD_3_68299:%.*]] = extractelement <4 x b8> [[TMP884]], i32 3
+; CHECK-NEXT: [[TMP888:%.*]] = bitcast b8 [[LOAD_3_68299]] to i8
+; CHECK-NEXT: [[PTR2_0_68:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 272
+; CHECK-NEXT: [[TMP889:%.*]] = bitcast i8 [[TMP885]] to b8
+; CHECK-NEXT: [[TMP890:%.*]] = insertelement <4 x b8> poison, b8 [[TMP889]], i32 0
+; CHECK-NEXT: [[TMP891:%.*]] = bitcast i8 [[TMP886]] to b8
+; CHECK-NEXT: [[TMP892:%.*]] = insertelement <4 x b8> [[TMP890]], b8 [[TMP891]], i32 1
+; CHECK-NEXT: [[TMP893:%.*]] = bitcast i8 [[TMP887]] to b8
+; CHECK-NEXT: [[TMP894:%.*]] = insertelement <4 x b8> [[TMP892]], b8 [[TMP893]], i32 2
+; CHECK-NEXT: [[TMP895:%.*]] = bitcast i8 [[TMP888]] to b8
+; CHECK-NEXT: [[TMP896:%.*]] = insertelement <4 x b8> [[TMP894]], b8 [[TMP895]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP896]], ptr [[PTR2_0_68]], align 4
+; CHECK-NEXT: [[PTR_0_69:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 276
+; CHECK-NEXT: [[TMP897:%.*]] = load <4 x b8>, ptr [[PTR_0_69]], align 4
+; CHECK-NEXT: [[LOAD_0_69300:%.*]] = extractelement <4 x b8> [[TMP897]], i32 0
+; CHECK-NEXT: [[TMP898:%.*]] = bitcast b8 [[LOAD_0_69300]] to i8
+; CHECK-NEXT: [[LOAD_1_69301:%.*]] = extractelement <4 x b8> [[TMP897]], i32 1
+; CHECK-NEXT: [[TMP899:%.*]] = bitcast b8 [[LOAD_1_69301]] to i8
+; CHECK-NEXT: [[LOAD_2_69302:%.*]] = extractelement <4 x b8> [[TMP897]], i32 2
+; CHECK-NEXT: [[TMP900:%.*]] = bitcast b8 [[LOAD_2_69302]] to i8
+; CHECK-NEXT: [[LOAD_3_69303:%.*]] = extractelement <4 x b8> [[TMP897]], i32 3
+; CHECK-NEXT: [[TMP901:%.*]] = bitcast b8 [[LOAD_3_69303]] to i8
+; CHECK-NEXT: [[PTR2_0_69:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 276
+; CHECK-NEXT: [[TMP902:%.*]] = bitcast i8 [[TMP898]] to b8
+; CHECK-NEXT: [[TMP903:%.*]] = insertelement <4 x b8> poison, b8 [[TMP902]], i32 0
+; CHECK-NEXT: [[TMP904:%.*]] = bitcast i8 [[TMP899]] to b8
+; CHECK-NEXT: [[TMP905:%.*]] = insertelement <4 x b8> [[TMP903]], b8 [[TMP904]], i32 1
+; CHECK-NEXT: [[TMP906:%.*]] = bitcast i8 [[TMP900]] to b8
+; CHECK-NEXT: [[TMP907:%.*]] = insertelement <4 x b8> [[TMP905]], b8 [[TMP906]], i32 2
+; CHECK-NEXT: [[TMP908:%.*]] = bitcast i8 [[TMP901]] to b8
+; CHECK-NEXT: [[TMP909:%.*]] = insertelement <4 x b8> [[TMP907]], b8 [[TMP908]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP909]], ptr [[PTR2_0_69]], align 4
+; CHECK-NEXT: [[PTR_0_70:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 280
+; CHECK-NEXT: [[TMP910:%.*]] = load <4 x b8>, ptr [[PTR_0_70]], align 4
+; CHECK-NEXT: [[LOAD_0_70304:%.*]] = extractelement <4 x b8> [[TMP910]], i32 0
+; CHECK-NEXT: [[TMP911:%.*]] = bitcast b8 [[LOAD_0_70304]] to i8
+; CHECK-NEXT: [[LOAD_1_70305:%.*]] = extractelement <4 x b8> [[TMP910]], i32 1
+; CHECK-NEXT: [[TMP912:%.*]] = bitcast b8 [[LOAD_1_70305]] to i8
+; CHECK-NEXT: [[LOAD_2_70306:%.*]] = extractelement <4 x b8> [[TMP910]], i32 2
+; CHECK-NEXT: [[TMP913:%.*]] = bitcast b8 [[LOAD_2_70306]] to i8
+; CHECK-NEXT: [[LOAD_3_70307:%.*]] = extractelement <4 x b8> [[TMP910]], i32 3
+; CHECK-NEXT: [[TMP914:%.*]] = bitcast b8 [[LOAD_3_70307]] to i8
+; CHECK-NEXT: [[PTR2_0_70:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 280
+; CHECK-NEXT: [[TMP915:%.*]] = bitcast i8 [[TMP911]] to b8
+; CHECK-NEXT: [[TMP916:%.*]] = insertelement <4 x b8> poison, b8 [[TMP915]], i32 0
+; CHECK-NEXT: [[TMP917:%.*]] = bitcast i8 [[TMP912]] to b8
+; CHECK-NEXT: [[TMP918:%.*]] = insertelement <4 x b8> [[TMP916]], b8 [[TMP917]], i32 1
+; CHECK-NEXT: [[TMP919:%.*]] = bitcast i8 [[TMP913]] to b8
+; CHECK-NEXT: [[TMP920:%.*]] = insertelement <4 x b8> [[TMP918]], b8 [[TMP919]], i32 2
+; CHECK-NEXT: [[TMP921:%.*]] = bitcast i8 [[TMP914]] to b8
+; CHECK-NEXT: [[TMP922:%.*]] = insertelement <4 x b8> [[TMP920]], b8 [[TMP921]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP922]], ptr [[PTR2_0_70]], align 4
+; CHECK-NEXT: [[PTR_0_71:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 284
+; CHECK-NEXT: [[TMP923:%.*]] = load <4 x b8>, ptr [[PTR_0_71]], align 4
+; CHECK-NEXT: [[LOAD_0_71308:%.*]] = extractelement <4 x b8> [[TMP923]], i32 0
+; CHECK-NEXT: [[TMP924:%.*]] = bitcast b8 [[LOAD_0_71308]] to i8
+; CHECK-NEXT: [[LOAD_1_71309:%.*]] = extractelement <4 x b8> [[TMP923]], i32 1
+; CHECK-NEXT: [[TMP925:%.*]] = bitcast b8 [[LOAD_1_71309]] to i8
+; CHECK-NEXT: [[LOAD_2_71310:%.*]] = extractelement <4 x b8> [[TMP923]], i32 2
+; CHECK-NEXT: [[TMP926:%.*]] = bitcast b8 [[LOAD_2_71310]] to i8
+; CHECK-NEXT: [[LOAD_3_71311:%.*]] = extractelement <4 x b8> [[TMP923]], i32 3
+; CHECK-NEXT: [[TMP927:%.*]] = bitcast b8 [[LOAD_3_71311]] to i8
+; CHECK-NEXT: [[PTR2_0_71:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 284
+; CHECK-NEXT: [[TMP928:%.*]] = bitcast i8 [[TMP924]] to b8
+; CHECK-NEXT: [[TMP929:%.*]] = insertelement <4 x b8> poison, b8 [[TMP928]], i32 0
+; CHECK-NEXT: [[TMP930:%.*]] = bitcast i8 [[TMP925]] to b8
+; CHECK-NEXT: [[TMP931:%.*]] = insertelement <4 x b8> [[TMP929]], b8 [[TMP930]], i32 1
+; CHECK-NEXT: [[TMP932:%.*]] = bitcast i8 [[TMP926]] to b8
+; CHECK-NEXT: [[TMP933:%.*]] = insertelement <4 x b8> [[TMP931]], b8 [[TMP932]], i32 2
+; CHECK-NEXT: [[TMP934:%.*]] = bitcast i8 [[TMP927]] to b8
+; CHECK-NEXT: [[TMP935:%.*]] = insertelement <4 x b8> [[TMP933]], b8 [[TMP934]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP935]], ptr [[PTR2_0_71]], align 4
+; CHECK-NEXT: [[PTR_0_72:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 288
+; CHECK-NEXT: [[TMP936:%.*]] = load <4 x b8>, ptr [[PTR_0_72]], align 4
+; CHECK-NEXT: [[LOAD_0_72312:%.*]] = extractelement <4 x b8> [[TMP936]], i32 0
+; CHECK-NEXT: [[TMP937:%.*]] = bitcast b8 [[LOAD_0_72312]] to i8
+; CHECK-NEXT: [[LOAD_1_72313:%.*]] = extractelement <4 x b8> [[TMP936]], i32 1
+; CHECK-NEXT: [[TMP938:%.*]] = bitcast b8 [[LOAD_1_72313]] to i8
+; CHECK-NEXT: [[LOAD_2_72314:%.*]] = extractelement <4 x b8> [[TMP936]], i32 2
+; CHECK-NEXT: [[TMP939:%.*]] = bitcast b8 [[LOAD_2_72314]] to i8
+; CHECK-NEXT: [[LOAD_3_72315:%.*]] = extractelement <4 x b8> [[TMP936]], i32 3
+; CHECK-NEXT: [[TMP940:%.*]] = bitcast b8 [[LOAD_3_72315]] to i8
+; CHECK-NEXT: [[PTR2_0_72:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 288
+; CHECK-NEXT: [[TMP941:%.*]] = bitcast i8 [[TMP937]] to b8
+; CHECK-NEXT: [[TMP942:%.*]] = insertelement <4 x b8> poison, b8 [[TMP941]], i32 0
+; CHECK-NEXT: [[TMP943:%.*]] = bitcast i8 [[TMP938]] to b8
+; CHECK-NEXT: [[TMP944:%.*]] = insertelement <4 x b8> [[TMP942]], b8 [[TMP943]], i32 1
+; CHECK-NEXT: [[TMP945:%.*]] = bitcast i8 [[TMP939]] to b8
+; CHECK-NEXT: [[TMP946:%.*]] = insertelement <4 x b8> [[TMP944]], b8 [[TMP945]], i32 2
+; CHECK-NEXT: [[TMP947:%.*]] = bitcast i8 [[TMP940]] to b8
+; CHECK-NEXT: [[TMP948:%.*]] = insertelement <4 x b8> [[TMP946]], b8 [[TMP947]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP948]], ptr [[PTR2_0_72]], align 4
+; CHECK-NEXT: [[PTR_0_73:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 292
+; CHECK-NEXT: [[TMP949:%.*]] = load <4 x b8>, ptr [[PTR_0_73]], align 4
+; CHECK-NEXT: [[LOAD_0_73316:%.*]] = extractelement <4 x b8> [[TMP949]], i32 0
+; CHECK-NEXT: [[TMP950:%.*]] = bitcast b8 [[LOAD_0_73316]] to i8
+; CHECK-NEXT: [[LOAD_1_73317:%.*]] = extractelement <4 x b8> [[TMP949]], i32 1
+; CHECK-NEXT: [[TMP951:%.*]] = bitcast b8 [[LOAD_1_73317]] to i8
+; CHECK-NEXT: [[LOAD_2_73318:%.*]] = extractelement <4 x b8> [[TMP949]], i32 2
+; CHECK-NEXT: [[TMP952:%.*]] = bitcast b8 [[LOAD_2_73318]] to i8
+; CHECK-NEXT: [[LOAD_3_73319:%.*]] = extractelement <4 x b8> [[TMP949]], i32 3
+; CHECK-NEXT: [[TMP953:%.*]] = bitcast b8 [[LOAD_3_73319]] to i8
+; CHECK-NEXT: [[PTR2_0_73:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 292
+; CHECK-NEXT: [[TMP954:%.*]] = bitcast i8 [[TMP950]] to b8
+; CHECK-NEXT: [[TMP955:%.*]] = insertelement <4 x b8> poison, b8 [[TMP954]], i32 0
+; CHECK-NEXT: [[TMP956:%.*]] = bitcast i8 [[TMP951]] to b8
+; CHECK-NEXT: [[TMP957:%.*]] = insertelement <4 x b8> [[TMP955]], b8 [[TMP956]], i32 1
+; CHECK-NEXT: [[TMP958:%.*]] = bitcast i8 [[TMP952]] to b8
+; CHECK-NEXT: [[TMP959:%.*]] = insertelement <4 x b8> [[TMP957]], b8 [[TMP958]], i32 2
+; CHECK-NEXT: [[TMP960:%.*]] = bitcast i8 [[TMP953]] to b8
+; CHECK-NEXT: [[TMP961:%.*]] = insertelement <4 x b8> [[TMP959]], b8 [[TMP960]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP961]], ptr [[PTR2_0_73]], align 4
+; CHECK-NEXT: [[PTR_0_74:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 296
+; CHECK-NEXT: [[TMP962:%.*]] = load <4 x b8>, ptr [[PTR_0_74]], align 4
+; CHECK-NEXT: [[LOAD_0_74320:%.*]] = extractelement <4 x b8> [[TMP962]], i32 0
+; CHECK-NEXT: [[TMP963:%.*]] = bitcast b8 [[LOAD_0_74320]] to i8
+; CHECK-NEXT: [[LOAD_1_74321:%.*]] = extractelement <4 x b8> [[TMP962]], i32 1
+; CHECK-NEXT: [[TMP964:%.*]] = bitcast b8 [[LOAD_1_74321]] to i8
+; CHECK-NEXT: [[LOAD_2_74322:%.*]] = extractelement <4 x b8> [[TMP962]], i32 2
+; CHECK-NEXT: [[TMP965:%.*]] = bitcast b8 [[LOAD_2_74322]] to i8
+; CHECK-NEXT: [[LOAD_3_74323:%.*]] = extractelement <4 x b8> [[TMP962]], i32 3
+; CHECK-NEXT: [[TMP966:%.*]] = bitcast b8 [[LOAD_3_74323]] to i8
+; CHECK-NEXT: [[PTR2_0_74:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 296
+; CHECK-NEXT: [[TMP967:%.*]] = bitcast i8 [[TMP963]] to b8
+; CHECK-NEXT: [[TMP968:%.*]] = insertelement <4 x b8> poison, b8 [[TMP967]], i32 0
+; CHECK-NEXT: [[TMP969:%.*]] = bitcast i8 [[TMP964]] to b8
+; CHECK-NEXT: [[TMP970:%.*]] = insertelement <4 x b8> [[TMP968]], b8 [[TMP969]], i32 1
+; CHECK-NEXT: [[TMP971:%.*]] = bitcast i8 [[TMP965]] to b8
+; CHECK-NEXT: [[TMP972:%.*]] = insertelement <4 x b8> [[TMP970]], b8 [[TMP971]], i32 2
+; CHECK-NEXT: [[TMP973:%.*]] = bitcast i8 [[TMP966]] to b8
+; CHECK-NEXT: [[TMP974:%.*]] = insertelement <4 x b8> [[TMP972]], b8 [[TMP973]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP974]], ptr [[PTR2_0_74]], align 4
+; CHECK-NEXT: [[PTR_0_75:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 300
+; CHECK-NEXT: [[TMP975:%.*]] = load <4 x b8>, ptr [[PTR_0_75]], align 4
+; CHECK-NEXT: [[LOAD_0_75324:%.*]] = extractelement <4 x b8> [[TMP975]], i32 0
+; CHECK-NEXT: [[TMP976:%.*]] = bitcast b8 [[LOAD_0_75324]] to i8
+; CHECK-NEXT: [[LOAD_1_75325:%.*]] = extractelement <4 x b8> [[TMP975]], i32 1
+; CHECK-NEXT: [[TMP977:%.*]] = bitcast b8 [[LOAD_1_75325]] to i8
+; CHECK-NEXT: [[LOAD_2_75326:%.*]] = extractelement <4 x b8> [[TMP975]], i32 2
+; CHECK-NEXT: [[TMP978:%.*]] = bitcast b8 [[LOAD_2_75326]] to i8
+; CHECK-NEXT: [[LOAD_3_75327:%.*]] = extractelement <4 x b8> [[TMP975]], i32 3
+; CHECK-NEXT: [[TMP979:%.*]] = bitcast b8 [[LOAD_3_75327]] to i8
+; CHECK-NEXT: [[PTR2_0_75:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 300
+; CHECK-NEXT: [[TMP980:%.*]] = bitcast i8 [[TMP976]] to b8
+; CHECK-NEXT: [[TMP981:%.*]] = insertelement <4 x b8> poison, b8 [[TMP980]], i32 0
+; CHECK-NEXT: [[TMP982:%.*]] = bitcast i8 [[TMP977]] to b8
+; CHECK-NEXT: [[TMP983:%.*]] = insertelement <4 x b8> [[TMP981]], b8 [[TMP982]], i32 1
+; CHECK-NEXT: [[TMP984:%.*]] = bitcast i8 [[TMP978]] to b8
+; CHECK-NEXT: [[TMP985:%.*]] = insertelement <4 x b8> [[TMP983]], b8 [[TMP984]], i32 2
+; CHECK-NEXT: [[TMP986:%.*]] = bitcast i8 [[TMP979]] to b8
+; CHECK-NEXT: [[TMP987:%.*]] = insertelement <4 x b8> [[TMP985]], b8 [[TMP986]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP987]], ptr [[PTR2_0_75]], align 4
+; CHECK-NEXT: [[PTR_0_76:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 304
+; CHECK-NEXT: [[TMP988:%.*]] = load <4 x b8>, ptr [[PTR_0_76]], align 4
+; CHECK-NEXT: [[LOAD_0_76328:%.*]] = extractelement <4 x b8> [[TMP988]], i32 0
+; CHECK-NEXT: [[TMP989:%.*]] = bitcast b8 [[LOAD_0_76328]] to i8
+; CHECK-NEXT: [[LOAD_1_76329:%.*]] = extractelement <4 x b8> [[TMP988]], i32 1
+; CHECK-NEXT: [[TMP990:%.*]] = bitcast b8 [[LOAD_1_76329]] to i8
+; CHECK-NEXT: [[LOAD_2_76330:%.*]] = extractelement <4 x b8> [[TMP988]], i32 2
+; CHECK-NEXT: [[TMP991:%.*]] = bitcast b8 [[LOAD_2_76330]] to i8
+; CHECK-NEXT: [[LOAD_3_76331:%.*]] = extractelement <4 x b8> [[TMP988]], i32 3
+; CHECK-NEXT: [[TMP992:%.*]] = bitcast b8 [[LOAD_3_76331]] to i8
+; CHECK-NEXT: [[PTR2_0_76:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 304
+; CHECK-NEXT: [[TMP993:%.*]] = bitcast i8 [[TMP989]] to b8
+; CHECK-NEXT: [[TMP994:%.*]] = insertelement <4 x b8> poison, b8 [[TMP993]], i32 0
+; CHECK-NEXT: [[TMP995:%.*]] = bitcast i8 [[TMP990]] to b8
+; CHECK-NEXT: [[TMP996:%.*]] = insertelement <4 x b8> [[TMP994]], b8 [[TMP995]], i32 1
+; CHECK-NEXT: [[TMP997:%.*]] = bitcast i8 [[TMP991]] to b8
+; CHECK-NEXT: [[TMP998:%.*]] = insertelement <4 x b8> [[TMP996]], b8 [[TMP997]], i32 2
+; CHECK-NEXT: [[TMP999:%.*]] = bitcast i8 [[TMP992]] to b8
+; CHECK-NEXT: [[TMP1000:%.*]] = insertelement <4 x b8> [[TMP998]], b8 [[TMP999]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1000]], ptr [[PTR2_0_76]], align 4
+; CHECK-NEXT: [[PTR_0_77:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 308
+; CHECK-NEXT: [[TMP1001:%.*]] = load <4 x b8>, ptr [[PTR_0_77]], align 4
+; CHECK-NEXT: [[LOAD_0_77332:%.*]] = extractelement <4 x b8> [[TMP1001]], i32 0
+; CHECK-NEXT: [[TMP1002:%.*]] = bitcast b8 [[LOAD_0_77332]] to i8
+; CHECK-NEXT: [[LOAD_1_77333:%.*]] = extractelement <4 x b8> [[TMP1001]], i32 1
+; CHECK-NEXT: [[TMP1003:%.*]] = bitcast b8 [[LOAD_1_77333]] to i8
+; CHECK-NEXT: [[LOAD_2_77334:%.*]] = extractelement <4 x b8> [[TMP1001]], i32 2
+; CHECK-NEXT: [[TMP1004:%.*]] = bitcast b8 [[LOAD_2_77334]] to i8
+; CHECK-NEXT: [[LOAD_3_77335:%.*]] = extractelement <4 x b8> [[TMP1001]], i32 3
+; CHECK-NEXT: [[TMP1005:%.*]] = bitcast b8 [[LOAD_3_77335]] to i8
+; CHECK-NEXT: [[PTR2_0_77:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 308
+; CHECK-NEXT: [[TMP1006:%.*]] = bitcast i8 [[TMP1002]] to b8
+; CHECK-NEXT: [[TMP1007:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1006]], i32 0
+; CHECK-NEXT: [[TMP1008:%.*]] = bitcast i8 [[TMP1003]] to b8
+; CHECK-NEXT: [[TMP1009:%.*]] = insertelement <4 x b8> [[TMP1007]], b8 [[TMP1008]], i32 1
+; CHECK-NEXT: [[TMP1010:%.*]] = bitcast i8 [[TMP1004]] to b8
+; CHECK-NEXT: [[TMP1011:%.*]] = insertelement <4 x b8> [[TMP1009]], b8 [[TMP1010]], i32 2
+; CHECK-NEXT: [[TMP1012:%.*]] = bitcast i8 [[TMP1005]] to b8
+; CHECK-NEXT: [[TMP1013:%.*]] = insertelement <4 x b8> [[TMP1011]], b8 [[TMP1012]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1013]], ptr [[PTR2_0_77]], align 4
+; CHECK-NEXT: [[PTR_0_78:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 312
+; CHECK-NEXT: [[TMP1014:%.*]] = load <4 x b8>, ptr [[PTR_0_78]], align 4
+; CHECK-NEXT: [[LOAD_0_78336:%.*]] = extractelement <4 x b8> [[TMP1014]], i32 0
+; CHECK-NEXT: [[TMP1015:%.*]] = bitcast b8 [[LOAD_0_78336]] to i8
+; CHECK-NEXT: [[LOAD_1_78337:%.*]] = extractelement <4 x b8> [[TMP1014]], i32 1
+; CHECK-NEXT: [[TMP1016:%.*]] = bitcast b8 [[LOAD_1_78337]] to i8
+; CHECK-NEXT: [[LOAD_2_78338:%.*]] = extractelement <4 x b8> [[TMP1014]], i32 2
+; CHECK-NEXT: [[TMP1017:%.*]] = bitcast b8 [[LOAD_2_78338]] to i8
+; CHECK-NEXT: [[LOAD_3_78339:%.*]] = extractelement <4 x b8> [[TMP1014]], i32 3
+; CHECK-NEXT: [[TMP1018:%.*]] = bitcast b8 [[LOAD_3_78339]] to i8
+; CHECK-NEXT: [[PTR2_0_78:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 312
+; CHECK-NEXT: [[TMP1019:%.*]] = bitcast i8 [[TMP1015]] to b8
+; CHECK-NEXT: [[TMP1020:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1019]], i32 0
+; CHECK-NEXT: [[TMP1021:%.*]] = bitcast i8 [[TMP1016]] to b8
+; CHECK-NEXT: [[TMP1022:%.*]] = insertelement <4 x b8> [[TMP1020]], b8 [[TMP1021]], i32 1
+; CHECK-NEXT: [[TMP1023:%.*]] = bitcast i8 [[TMP1017]] to b8
+; CHECK-NEXT: [[TMP1024:%.*]] = insertelement <4 x b8> [[TMP1022]], b8 [[TMP1023]], i32 2
+; CHECK-NEXT: [[TMP1025:%.*]] = bitcast i8 [[TMP1018]] to b8
+; CHECK-NEXT: [[TMP1026:%.*]] = insertelement <4 x b8> [[TMP1024]], b8 [[TMP1025]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1026]], ptr [[PTR2_0_78]], align 4
+; CHECK-NEXT: [[PTR_0_79:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 316
+; CHECK-NEXT: [[TMP1027:%.*]] = load <4 x b8>, ptr [[PTR_0_79]], align 4
+; CHECK-NEXT: [[LOAD_0_79340:%.*]] = extractelement <4 x b8> [[TMP1027]], i32 0
+; CHECK-NEXT: [[TMP1028:%.*]] = bitcast b8 [[LOAD_0_79340]] to i8
+; CHECK-NEXT: [[LOAD_1_79341:%.*]] = extractelement <4 x b8> [[TMP1027]], i32 1
+; CHECK-NEXT: [[TMP1029:%.*]] = bitcast b8 [[LOAD_1_79341]] to i8
+; CHECK-NEXT: [[LOAD_2_79342:%.*]] = extractelement <4 x b8> [[TMP1027]], i32 2
+; CHECK-NEXT: [[TMP1030:%.*]] = bitcast b8 [[LOAD_2_79342]] to i8
+; CHECK-NEXT: [[LOAD_3_79343:%.*]] = extractelement <4 x b8> [[TMP1027]], i32 3
+; CHECK-NEXT: [[TMP1031:%.*]] = bitcast b8 [[LOAD_3_79343]] to i8
+; CHECK-NEXT: [[PTR2_0_79:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 316
+; CHECK-NEXT: [[TMP1032:%.*]] = bitcast i8 [[TMP1028]] to b8
+; CHECK-NEXT: [[TMP1033:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1032]], i32 0
+; CHECK-NEXT: [[TMP1034:%.*]] = bitcast i8 [[TMP1029]] to b8
+; CHECK-NEXT: [[TMP1035:%.*]] = insertelement <4 x b8> [[TMP1033]], b8 [[TMP1034]], i32 1
+; CHECK-NEXT: [[TMP1036:%.*]] = bitcast i8 [[TMP1030]] to b8
+; CHECK-NEXT: [[TMP1037:%.*]] = insertelement <4 x b8> [[TMP1035]], b8 [[TMP1036]], i32 2
+; CHECK-NEXT: [[TMP1038:%.*]] = bitcast i8 [[TMP1031]] to b8
+; CHECK-NEXT: [[TMP1039:%.*]] = insertelement <4 x b8> [[TMP1037]], b8 [[TMP1038]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1039]], ptr [[PTR2_0_79]], align 4
+; CHECK-NEXT: [[PTR_0_80:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 320
+; CHECK-NEXT: [[TMP1040:%.*]] = load <4 x b8>, ptr [[PTR_0_80]], align 4
+; CHECK-NEXT: [[LOAD_0_80344:%.*]] = extractelement <4 x b8> [[TMP1040]], i32 0
+; CHECK-NEXT: [[TMP1041:%.*]] = bitcast b8 [[LOAD_0_80344]] to i8
+; CHECK-NEXT: [[LOAD_1_80345:%.*]] = extractelement <4 x b8> [[TMP1040]], i32 1
+; CHECK-NEXT: [[TMP1042:%.*]] = bitcast b8 [[LOAD_1_80345]] to i8
+; CHECK-NEXT: [[LOAD_2_80346:%.*]] = extractelement <4 x b8> [[TMP1040]], i32 2
+; CHECK-NEXT: [[TMP1043:%.*]] = bitcast b8 [[LOAD_2_80346]] to i8
+; CHECK-NEXT: [[LOAD_3_80347:%.*]] = extractelement <4 x b8> [[TMP1040]], i32 3
+; CHECK-NEXT: [[TMP1044:%.*]] = bitcast b8 [[LOAD_3_80347]] to i8
+; CHECK-NEXT: [[PTR2_0_80:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 320
+; CHECK-NEXT: [[TMP1045:%.*]] = bitcast i8 [[TMP1041]] to b8
+; CHECK-NEXT: [[TMP1046:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1045]], i32 0
+; CHECK-NEXT: [[TMP1047:%.*]] = bitcast i8 [[TMP1042]] to b8
+; CHECK-NEXT: [[TMP1048:%.*]] = insertelement <4 x b8> [[TMP1046]], b8 [[TMP1047]], i32 1
+; CHECK-NEXT: [[TMP1049:%.*]] = bitcast i8 [[TMP1043]] to b8
+; CHECK-NEXT: [[TMP1050:%.*]] = insertelement <4 x b8> [[TMP1048]], b8 [[TMP1049]], i32 2
+; CHECK-NEXT: [[TMP1051:%.*]] = bitcast i8 [[TMP1044]] to b8
+; CHECK-NEXT: [[TMP1052:%.*]] = insertelement <4 x b8> [[TMP1050]], b8 [[TMP1051]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1052]], ptr [[PTR2_0_80]], align 4
+; CHECK-NEXT: [[PTR_0_81:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 324
+; CHECK-NEXT: [[TMP1053:%.*]] = load <4 x b8>, ptr [[PTR_0_81]], align 4
+; CHECK-NEXT: [[LOAD_0_81348:%.*]] = extractelement <4 x b8> [[TMP1053]], i32 0
+; CHECK-NEXT: [[TMP1054:%.*]] = bitcast b8 [[LOAD_0_81348]] to i8
+; CHECK-NEXT: [[LOAD_1_81349:%.*]] = extractelement <4 x b8> [[TMP1053]], i32 1
+; CHECK-NEXT: [[TMP1055:%.*]] = bitcast b8 [[LOAD_1_81349]] to i8
+; CHECK-NEXT: [[LOAD_2_81350:%.*]] = extractelement <4 x b8> [[TMP1053]], i32 2
+; CHECK-NEXT: [[TMP1056:%.*]] = bitcast b8 [[LOAD_2_81350]] to i8
+; CHECK-NEXT: [[LOAD_3_81351:%.*]] = extractelement <4 x b8> [[TMP1053]], i32 3
+; CHECK-NEXT: [[TMP1057:%.*]] = bitcast b8 [[LOAD_3_81351]] to i8
+; CHECK-NEXT: [[PTR2_0_81:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 324
+; CHECK-NEXT: [[TMP1058:%.*]] = bitcast i8 [[TMP1054]] to b8
+; CHECK-NEXT: [[TMP1059:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1058]], i32 0
+; CHECK-NEXT: [[TMP1060:%.*]] = bitcast i8 [[TMP1055]] to b8
+; CHECK-NEXT: [[TMP1061:%.*]] = insertelement <4 x b8> [[TMP1059]], b8 [[TMP1060]], i32 1
+; CHECK-NEXT: [[TMP1062:%.*]] = bitcast i8 [[TMP1056]] to b8
+; CHECK-NEXT: [[TMP1063:%.*]] = insertelement <4 x b8> [[TMP1061]], b8 [[TMP1062]], i32 2
+; CHECK-NEXT: [[TMP1064:%.*]] = bitcast i8 [[TMP1057]] to b8
+; CHECK-NEXT: [[TMP1065:%.*]] = insertelement <4 x b8> [[TMP1063]], b8 [[TMP1064]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1065]], ptr [[PTR2_0_81]], align 4
+; CHECK-NEXT: [[PTR_0_82:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 328
+; CHECK-NEXT: [[TMP1066:%.*]] = load <4 x b8>, ptr [[PTR_0_82]], align 4
+; CHECK-NEXT: [[LOAD_0_82352:%.*]] = extractelement <4 x b8> [[TMP1066]], i32 0
+; CHECK-NEXT: [[TMP1067:%.*]] = bitcast b8 [[LOAD_0_82352]] to i8
+; CHECK-NEXT: [[LOAD_1_82353:%.*]] = extractelement <4 x b8> [[TMP1066]], i32 1
+; CHECK-NEXT: [[TMP1068:%.*]] = bitcast b8 [[LOAD_1_82353]] to i8
+; CHECK-NEXT: [[LOAD_2_82354:%.*]] = extractelement <4 x b8> [[TMP1066]], i32 2
+; CHECK-NEXT: [[TMP1069:%.*]] = bitcast b8 [[LOAD_2_82354]] to i8
+; CHECK-NEXT: [[LOAD_3_82355:%.*]] = extractelement <4 x b8> [[TMP1066]], i32 3
+; CHECK-NEXT: [[TMP1070:%.*]] = bitcast b8 [[LOAD_3_82355]] to i8
+; CHECK-NEXT: [[PTR2_0_82:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 328
+; CHECK-NEXT: [[TMP1071:%.*]] = bitcast i8 [[TMP1067]] to b8
+; CHECK-NEXT: [[TMP1072:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1071]], i32 0
+; CHECK-NEXT: [[TMP1073:%.*]] = bitcast i8 [[TMP1068]] to b8
+; CHECK-NEXT: [[TMP1074:%.*]] = insertelement <4 x b8> [[TMP1072]], b8 [[TMP1073]], i32 1
+; CHECK-NEXT: [[TMP1075:%.*]] = bitcast i8 [[TMP1069]] to b8
+; CHECK-NEXT: [[TMP1076:%.*]] = insertelement <4 x b8> [[TMP1074]], b8 [[TMP1075]], i32 2
+; CHECK-NEXT: [[TMP1077:%.*]] = bitcast i8 [[TMP1070]] to b8
+; CHECK-NEXT: [[TMP1078:%.*]] = insertelement <4 x b8> [[TMP1076]], b8 [[TMP1077]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1078]], ptr [[PTR2_0_82]], align 4
+; CHECK-NEXT: [[PTR_0_83:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 332
+; CHECK-NEXT: [[TMP1079:%.*]] = load <4 x b8>, ptr [[PTR_0_83]], align 4
+; CHECK-NEXT: [[LOAD_0_83356:%.*]] = extractelement <4 x b8> [[TMP1079]], i32 0
+; CHECK-NEXT: [[TMP1080:%.*]] = bitcast b8 [[LOAD_0_83356]] to i8
+; CHECK-NEXT: [[LOAD_1_83357:%.*]] = extractelement <4 x b8> [[TMP1079]], i32 1
+; CHECK-NEXT: [[TMP1081:%.*]] = bitcast b8 [[LOAD_1_83357]] to i8
+; CHECK-NEXT: [[LOAD_2_83358:%.*]] = extractelement <4 x b8> [[TMP1079]], i32 2
+; CHECK-NEXT: [[TMP1082:%.*]] = bitcast b8 [[LOAD_2_83358]] to i8
+; CHECK-NEXT: [[LOAD_3_83359:%.*]] = extractelement <4 x b8> [[TMP1079]], i32 3
+; CHECK-NEXT: [[TMP1083:%.*]] = bitcast b8 [[LOAD_3_83359]] to i8
+; CHECK-NEXT: [[PTR2_0_83:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 332
+; CHECK-NEXT: [[TMP1084:%.*]] = bitcast i8 [[TMP1080]] to b8
+; CHECK-NEXT: [[TMP1085:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1084]], i32 0
+; CHECK-NEXT: [[TMP1086:%.*]] = bitcast i8 [[TMP1081]] to b8
+; CHECK-NEXT: [[TMP1087:%.*]] = insertelement <4 x b8> [[TMP1085]], b8 [[TMP1086]], i32 1
+; CHECK-NEXT: [[TMP1088:%.*]] = bitcast i8 [[TMP1082]] to b8
+; CHECK-NEXT: [[TMP1089:%.*]] = insertelement <4 x b8> [[TMP1087]], b8 [[TMP1088]], i32 2
+; CHECK-NEXT: [[TMP1090:%.*]] = bitcast i8 [[TMP1083]] to b8
+; CHECK-NEXT: [[TMP1091:%.*]] = insertelement <4 x b8> [[TMP1089]], b8 [[TMP1090]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1091]], ptr [[PTR2_0_83]], align 4
+; CHECK-NEXT: [[PTR_0_84:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 336
+; CHECK-NEXT: [[TMP1092:%.*]] = load <4 x b8>, ptr [[PTR_0_84]], align 4
+; CHECK-NEXT: [[LOAD_0_84360:%.*]] = extractelement <4 x b8> [[TMP1092]], i32 0
+; CHECK-NEXT: [[TMP1093:%.*]] = bitcast b8 [[LOAD_0_84360]] to i8
+; CHECK-NEXT: [[LOAD_1_84361:%.*]] = extractelement <4 x b8> [[TMP1092]], i32 1
+; CHECK-NEXT: [[TMP1094:%.*]] = bitcast b8 [[LOAD_1_84361]] to i8
+; CHECK-NEXT: [[LOAD_2_84362:%.*]] = extractelement <4 x b8> [[TMP1092]], i32 2
+; CHECK-NEXT: [[TMP1095:%.*]] = bitcast b8 [[LOAD_2_84362]] to i8
+; CHECK-NEXT: [[LOAD_3_84363:%.*]] = extractelement <4 x b8> [[TMP1092]], i32 3
+; CHECK-NEXT: [[TMP1096:%.*]] = bitcast b8 [[LOAD_3_84363]] to i8
+; CHECK-NEXT: [[PTR2_0_84:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 336
+; CHECK-NEXT: [[TMP1097:%.*]] = bitcast i8 [[TMP1093]] to b8
+; CHECK-NEXT: [[TMP1098:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1097]], i32 0
+; CHECK-NEXT: [[TMP1099:%.*]] = bitcast i8 [[TMP1094]] to b8
+; CHECK-NEXT: [[TMP1100:%.*]] = insertelement <4 x b8> [[TMP1098]], b8 [[TMP1099]], i32 1
+; CHECK-NEXT: [[TMP1101:%.*]] = bitcast i8 [[TMP1095]] to b8
+; CHECK-NEXT: [[TMP1102:%.*]] = insertelement <4 x b8> [[TMP1100]], b8 [[TMP1101]], i32 2
+; CHECK-NEXT: [[TMP1103:%.*]] = bitcast i8 [[TMP1096]] to b8
+; CHECK-NEXT: [[TMP1104:%.*]] = insertelement <4 x b8> [[TMP1102]], b8 [[TMP1103]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1104]], ptr [[PTR2_0_84]], align 4
+; CHECK-NEXT: [[PTR_0_85:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 340
+; CHECK-NEXT: [[TMP1105:%.*]] = load <4 x b8>, ptr [[PTR_0_85]], align 4
+; CHECK-NEXT: [[LOAD_0_85364:%.*]] = extractelement <4 x b8> [[TMP1105]], i32 0
+; CHECK-NEXT: [[TMP1106:%.*]] = bitcast b8 [[LOAD_0_85364]] to i8
+; CHECK-NEXT: [[LOAD_1_85365:%.*]] = extractelement <4 x b8> [[TMP1105]], i32 1
+; CHECK-NEXT: [[TMP1107:%.*]] = bitcast b8 [[LOAD_1_85365]] to i8
+; CHECK-NEXT: [[LOAD_2_85366:%.*]] = extractelement <4 x b8> [[TMP1105]], i32 2
+; CHECK-NEXT: [[TMP1108:%.*]] = bitcast b8 [[LOAD_2_85366]] to i8
+; CHECK-NEXT: [[LOAD_3_85367:%.*]] = extractelement <4 x b8> [[TMP1105]], i32 3
+; CHECK-NEXT: [[TMP1109:%.*]] = bitcast b8 [[LOAD_3_85367]] to i8
+; CHECK-NEXT: [[PTR2_0_85:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 340
+; CHECK-NEXT: [[TMP1110:%.*]] = bitcast i8 [[TMP1106]] to b8
+; CHECK-NEXT: [[TMP1111:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1110]], i32 0
+; CHECK-NEXT: [[TMP1112:%.*]] = bitcast i8 [[TMP1107]] to b8
+; CHECK-NEXT: [[TMP1113:%.*]] = insertelement <4 x b8> [[TMP1111]], b8 [[TMP1112]], i32 1
+; CHECK-NEXT: [[TMP1114:%.*]] = bitcast i8 [[TMP1108]] to b8
+; CHECK-NEXT: [[TMP1115:%.*]] = insertelement <4 x b8> [[TMP1113]], b8 [[TMP1114]], i32 2
+; CHECK-NEXT: [[TMP1116:%.*]] = bitcast i8 [[TMP1109]] to b8
+; CHECK-NEXT: [[TMP1117:%.*]] = insertelement <4 x b8> [[TMP1115]], b8 [[TMP1116]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1117]], ptr [[PTR2_0_85]], align 4
+; CHECK-NEXT: [[PTR_0_86:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 344
+; CHECK-NEXT: [[TMP1118:%.*]] = load <4 x b8>, ptr [[PTR_0_86]], align 4
+; CHECK-NEXT: [[LOAD_0_86368:%.*]] = extractelement <4 x b8> [[TMP1118]], i32 0
+; CHECK-NEXT: [[TMP1119:%.*]] = bitcast b8 [[LOAD_0_86368]] to i8
+; CHECK-NEXT: [[LOAD_1_86369:%.*]] = extractelement <4 x b8> [[TMP1118]], i32 1
+; CHECK-NEXT: [[TMP1120:%.*]] = bitcast b8 [[LOAD_1_86369]] to i8
+; CHECK-NEXT: [[LOAD_2_86370:%.*]] = extractelement <4 x b8> [[TMP1118]], i32 2
+; CHECK-NEXT: [[TMP1121:%.*]] = bitcast b8 [[LOAD_2_86370]] to i8
+; CHECK-NEXT: [[LOAD_3_86371:%.*]] = extractelement <4 x b8> [[TMP1118]], i32 3
+; CHECK-NEXT: [[TMP1122:%.*]] = bitcast b8 [[LOAD_3_86371]] to i8
+; CHECK-NEXT: [[PTR2_0_86:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 344
+; CHECK-NEXT: [[TMP1123:%.*]] = bitcast i8 [[TMP1119]] to b8
+; CHECK-NEXT: [[TMP1124:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1123]], i32 0
+; CHECK-NEXT: [[TMP1125:%.*]] = bitcast i8 [[TMP1120]] to b8
+; CHECK-NEXT: [[TMP1126:%.*]] = insertelement <4 x b8> [[TMP1124]], b8 [[TMP1125]], i32 1
+; CHECK-NEXT: [[TMP1127:%.*]] = bitcast i8 [[TMP1121]] to b8
+; CHECK-NEXT: [[TMP1128:%.*]] = insertelement <4 x b8> [[TMP1126]], b8 [[TMP1127]], i32 2
+; CHECK-NEXT: [[TMP1129:%.*]] = bitcast i8 [[TMP1122]] to b8
+; CHECK-NEXT: [[TMP1130:%.*]] = insertelement <4 x b8> [[TMP1128]], b8 [[TMP1129]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1130]], ptr [[PTR2_0_86]], align 4
+; CHECK-NEXT: [[PTR_0_87:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 348
+; CHECK-NEXT: [[TMP1131:%.*]] = load <4 x b8>, ptr [[PTR_0_87]], align 4
+; CHECK-NEXT: [[LOAD_0_87372:%.*]] = extractelement <4 x b8> [[TMP1131]], i32 0
+; CHECK-NEXT: [[TMP1132:%.*]] = bitcast b8 [[LOAD_0_87372]] to i8
+; CHECK-NEXT: [[LOAD_1_87373:%.*]] = extractelement <4 x b8> [[TMP1131]], i32 1
+; CHECK-NEXT: [[TMP1133:%.*]] = bitcast b8 [[LOAD_1_87373]] to i8
+; CHECK-NEXT: [[LOAD_2_87374:%.*]] = extractelement <4 x b8> [[TMP1131]], i32 2
+; CHECK-NEXT: [[TMP1134:%.*]] = bitcast b8 [[LOAD_2_87374]] to i8
+; CHECK-NEXT: [[LOAD_3_87375:%.*]] = extractelement <4 x b8> [[TMP1131]], i32 3
+; CHECK-NEXT: [[TMP1135:%.*]] = bitcast b8 [[LOAD_3_87375]] to i8
+; CHECK-NEXT: [[PTR2_0_87:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 348
+; CHECK-NEXT: [[TMP1136:%.*]] = bitcast i8 [[TMP1132]] to b8
+; CHECK-NEXT: [[TMP1137:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1136]], i32 0
+; CHECK-NEXT: [[TMP1138:%.*]] = bitcast i8 [[TMP1133]] to b8
+; CHECK-NEXT: [[TMP1139:%.*]] = insertelement <4 x b8> [[TMP1137]], b8 [[TMP1138]], i32 1
+; CHECK-NEXT: [[TMP1140:%.*]] = bitcast i8 [[TMP1134]] to b8
+; CHECK-NEXT: [[TMP1141:%.*]] = insertelement <4 x b8> [[TMP1139]], b8 [[TMP1140]], i32 2
+; CHECK-NEXT: [[TMP1142:%.*]] = bitcast i8 [[TMP1135]] to b8
+; CHECK-NEXT: [[TMP1143:%.*]] = insertelement <4 x b8> [[TMP1141]], b8 [[TMP1142]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1143]], ptr [[PTR2_0_87]], align 4
+; CHECK-NEXT: [[PTR_0_88:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 352
+; CHECK-NEXT: [[TMP1144:%.*]] = load <4 x b8>, ptr [[PTR_0_88]], align 4
+; CHECK-NEXT: [[LOAD_0_88376:%.*]] = extractelement <4 x b8> [[TMP1144]], i32 0
+; CHECK-NEXT: [[TMP1145:%.*]] = bitcast b8 [[LOAD_0_88376]] to i8
+; CHECK-NEXT: [[LOAD_1_88377:%.*]] = extractelement <4 x b8> [[TMP1144]], i32 1
+; CHECK-NEXT: [[TMP1146:%.*]] = bitcast b8 [[LOAD_1_88377]] to i8
+; CHECK-NEXT: [[LOAD_2_88378:%.*]] = extractelement <4 x b8> [[TMP1144]], i32 2
+; CHECK-NEXT: [[TMP1147:%.*]] = bitcast b8 [[LOAD_2_88378]] to i8
+; CHECK-NEXT: [[LOAD_3_88379:%.*]] = extractelement <4 x b8> [[TMP1144]], i32 3
+; CHECK-NEXT: [[TMP1148:%.*]] = bitcast b8 [[LOAD_3_88379]] to i8
+; CHECK-NEXT: [[PTR2_0_88:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 352
+; CHECK-NEXT: [[TMP1149:%.*]] = bitcast i8 [[TMP1145]] to b8
+; CHECK-NEXT: [[TMP1150:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1149]], i32 0
+; CHECK-NEXT: [[TMP1151:%.*]] = bitcast i8 [[TMP1146]] to b8
+; CHECK-NEXT: [[TMP1152:%.*]] = insertelement <4 x b8> [[TMP1150]], b8 [[TMP1151]], i32 1
+; CHECK-NEXT: [[TMP1153:%.*]] = bitcast i8 [[TMP1147]] to b8
+; CHECK-NEXT: [[TMP1154:%.*]] = insertelement <4 x b8> [[TMP1152]], b8 [[TMP1153]], i32 2
+; CHECK-NEXT: [[TMP1155:%.*]] = bitcast i8 [[TMP1148]] to b8
+; CHECK-NEXT: [[TMP1156:%.*]] = insertelement <4 x b8> [[TMP1154]], b8 [[TMP1155]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1156]], ptr [[PTR2_0_88]], align 4
+; CHECK-NEXT: [[PTR_0_89:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 356
+; CHECK-NEXT: [[TMP1157:%.*]] = load <4 x b8>, ptr [[PTR_0_89]], align 4
+; CHECK-NEXT: [[LOAD_0_89380:%.*]] = extractelement <4 x b8> [[TMP1157]], i32 0
+; CHECK-NEXT: [[TMP1158:%.*]] = bitcast b8 [[LOAD_0_89380]] to i8
+; CHECK-NEXT: [[LOAD_1_89381:%.*]] = extractelement <4 x b8> [[TMP1157]], i32 1
+; CHECK-NEXT: [[TMP1159:%.*]] = bitcast b8 [[LOAD_1_89381]] to i8
+; CHECK-NEXT: [[LOAD_2_89382:%.*]] = extractelement <4 x b8> [[TMP1157]], i32 2
+; CHECK-NEXT: [[TMP1160:%.*]] = bitcast b8 [[LOAD_2_89382]] to i8
+; CHECK-NEXT: [[LOAD_3_89383:%.*]] = extractelement <4 x b8> [[TMP1157]], i32 3
+; CHECK-NEXT: [[TMP1161:%.*]] = bitcast b8 [[LOAD_3_89383]] to i8
+; CHECK-NEXT: [[PTR2_0_89:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 356
+; CHECK-NEXT: [[TMP1162:%.*]] = bitcast i8 [[TMP1158]] to b8
+; CHECK-NEXT: [[TMP1163:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1162]], i32 0
+; CHECK-NEXT: [[TMP1164:%.*]] = bitcast i8 [[TMP1159]] to b8
+; CHECK-NEXT: [[TMP1165:%.*]] = insertelement <4 x b8> [[TMP1163]], b8 [[TMP1164]], i32 1
+; CHECK-NEXT: [[TMP1166:%.*]] = bitcast i8 [[TMP1160]] to b8
+; CHECK-NEXT: [[TMP1167:%.*]] = insertelement <4 x b8> [[TMP1165]], b8 [[TMP1166]], i32 2
+; CHECK-NEXT: [[TMP1168:%.*]] = bitcast i8 [[TMP1161]] to b8
+; CHECK-NEXT: [[TMP1169:%.*]] = insertelement <4 x b8> [[TMP1167]], b8 [[TMP1168]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1169]], ptr [[PTR2_0_89]], align 4
+; CHECK-NEXT: [[PTR_0_90:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 360
+; CHECK-NEXT: [[TMP1170:%.*]] = load <4 x b8>, ptr [[PTR_0_90]], align 4
+; CHECK-NEXT: [[LOAD_0_90384:%.*]] = extractelement <4 x b8> [[TMP1170]], i32 0
+; CHECK-NEXT: [[TMP1171:%.*]] = bitcast b8 [[LOAD_0_90384]] to i8
+; CHECK-NEXT: [[LOAD_1_90385:%.*]] = extractelement <4 x b8> [[TMP1170]], i32 1
+; CHECK-NEXT: [[TMP1172:%.*]] = bitcast b8 [[LOAD_1_90385]] to i8
+; CHECK-NEXT: [[LOAD_2_90386:%.*]] = extractelement <4 x b8> [[TMP1170]], i32 2
+; CHECK-NEXT: [[TMP1173:%.*]] = bitcast b8 [[LOAD_2_90386]] to i8
+; CHECK-NEXT: [[LOAD_3_90387:%.*]] = extractelement <4 x b8> [[TMP1170]], i32 3
+; CHECK-NEXT: [[TMP1174:%.*]] = bitcast b8 [[LOAD_3_90387]] to i8
+; CHECK-NEXT: [[PTR2_0_90:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 360
+; CHECK-NEXT: [[TMP1175:%.*]] = bitcast i8 [[TMP1171]] to b8
+; CHECK-NEXT: [[TMP1176:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1175]], i32 0
+; CHECK-NEXT: [[TMP1177:%.*]] = bitcast i8 [[TMP1172]] to b8
+; CHECK-NEXT: [[TMP1178:%.*]] = insertelement <4 x b8> [[TMP1176]], b8 [[TMP1177]], i32 1
+; CHECK-NEXT: [[TMP1179:%.*]] = bitcast i8 [[TMP1173]] to b8
+; CHECK-NEXT: [[TMP1180:%.*]] = insertelement <4 x b8> [[TMP1178]], b8 [[TMP1179]], i32 2
+; CHECK-NEXT: [[TMP1181:%.*]] = bitcast i8 [[TMP1174]] to b8
+; CHECK-NEXT: [[TMP1182:%.*]] = insertelement <4 x b8> [[TMP1180]], b8 [[TMP1181]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1182]], ptr [[PTR2_0_90]], align 4
+; CHECK-NEXT: [[PTR_0_91:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 364
+; CHECK-NEXT: [[TMP1183:%.*]] = load <4 x b8>, ptr [[PTR_0_91]], align 4
+; CHECK-NEXT: [[LOAD_0_91388:%.*]] = extractelement <4 x b8> [[TMP1183]], i32 0
+; CHECK-NEXT: [[TMP1184:%.*]] = bitcast b8 [[LOAD_0_91388]] to i8
+; CHECK-NEXT: [[LOAD_1_91389:%.*]] = extractelement <4 x b8> [[TMP1183]], i32 1
+; CHECK-NEXT: [[TMP1185:%.*]] = bitcast b8 [[LOAD_1_91389]] to i8
+; CHECK-NEXT: [[LOAD_2_91390:%.*]] = extractelement <4 x b8> [[TMP1183]], i32 2
+; CHECK-NEXT: [[TMP1186:%.*]] = bitcast b8 [[LOAD_2_91390]] to i8
+; CHECK-NEXT: [[LOAD_3_91391:%.*]] = extractelement <4 x b8> [[TMP1183]], i32 3
+; CHECK-NEXT: [[TMP1187:%.*]] = bitcast b8 [[LOAD_3_91391]] to i8
+; CHECK-NEXT: [[PTR2_0_91:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 364
+; CHECK-NEXT: [[TMP1188:%.*]] = bitcast i8 [[TMP1184]] to b8
+; CHECK-NEXT: [[TMP1189:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1188]], i32 0
+; CHECK-NEXT: [[TMP1190:%.*]] = bitcast i8 [[TMP1185]] to b8
+; CHECK-NEXT: [[TMP1191:%.*]] = insertelement <4 x b8> [[TMP1189]], b8 [[TMP1190]], i32 1
+; CHECK-NEXT: [[TMP1192:%.*]] = bitcast i8 [[TMP1186]] to b8
+; CHECK-NEXT: [[TMP1193:%.*]] = insertelement <4 x b8> [[TMP1191]], b8 [[TMP1192]], i32 2
+; CHECK-NEXT: [[TMP1194:%.*]] = bitcast i8 [[TMP1187]] to b8
+; CHECK-NEXT: [[TMP1195:%.*]] = insertelement <4 x b8> [[TMP1193]], b8 [[TMP1194]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1195]], ptr [[PTR2_0_91]], align 4
+; CHECK-NEXT: [[PTR_0_92:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 368
+; CHECK-NEXT: [[TMP1196:%.*]] = load <4 x b8>, ptr [[PTR_0_92]], align 4
+; CHECK-NEXT: [[LOAD_0_92392:%.*]] = extractelement <4 x b8> [[TMP1196]], i32 0
+; CHECK-NEXT: [[TMP1197:%.*]] = bitcast b8 [[LOAD_0_92392]] to i8
+; CHECK-NEXT: [[LOAD_1_92393:%.*]] = extractelement <4 x b8> [[TMP1196]], i32 1
+; CHECK-NEXT: [[TMP1198:%.*]] = bitcast b8 [[LOAD_1_92393]] to i8
+; CHECK-NEXT: [[LOAD_2_92394:%.*]] = extractelement <4 x b8> [[TMP1196]], i32 2
+; CHECK-NEXT: [[TMP1199:%.*]] = bitcast b8 [[LOAD_2_92394]] to i8
+; CHECK-NEXT: [[LOAD_3_92395:%.*]] = extractelement <4 x b8> [[TMP1196]], i32 3
+; CHECK-NEXT: [[TMP1200:%.*]] = bitcast b8 [[LOAD_3_92395]] to i8
+; CHECK-NEXT: [[PTR2_0_92:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 368
+; CHECK-NEXT: [[TMP1201:%.*]] = bitcast i8 [[TMP1197]] to b8
+; CHECK-NEXT: [[TMP1202:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1201]], i32 0
+; CHECK-NEXT: [[TMP1203:%.*]] = bitcast i8 [[TMP1198]] to b8
+; CHECK-NEXT: [[TMP1204:%.*]] = insertelement <4 x b8> [[TMP1202]], b8 [[TMP1203]], i32 1
+; CHECK-NEXT: [[TMP1205:%.*]] = bitcast i8 [[TMP1199]] to b8
+; CHECK-NEXT: [[TMP1206:%.*]] = insertelement <4 x b8> [[TMP1204]], b8 [[TMP1205]], i32 2
+; CHECK-NEXT: [[TMP1207:%.*]] = bitcast i8 [[TMP1200]] to b8
+; CHECK-NEXT: [[TMP1208:%.*]] = insertelement <4 x b8> [[TMP1206]], b8 [[TMP1207]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1208]], ptr [[PTR2_0_92]], align 4
+; CHECK-NEXT: [[PTR_0_93:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 372
+; CHECK-NEXT: [[TMP1209:%.*]] = load <4 x b8>, ptr [[PTR_0_93]], align 4
+; CHECK-NEXT: [[LOAD_0_93396:%.*]] = extractelement <4 x b8> [[TMP1209]], i32 0
+; CHECK-NEXT: [[TMP1210:%.*]] = bitcast b8 [[LOAD_0_93396]] to i8
+; CHECK-NEXT: [[LOAD_1_93397:%.*]] = extractelement <4 x b8> [[TMP1209]], i32 1
+; CHECK-NEXT: [[TMP1211:%.*]] = bitcast b8 [[LOAD_1_93397]] to i8
+; CHECK-NEXT: [[LOAD_2_93398:%.*]] = extractelement <4 x b8> [[TMP1209]], i32 2
+; CHECK-NEXT: [[TMP1212:%.*]] = bitcast b8 [[LOAD_2_93398]] to i8
+; CHECK-NEXT: [[LOAD_3_93399:%.*]] = extractelement <4 x b8> [[TMP1209]], i32 3
+; CHECK-NEXT: [[TMP1213:%.*]] = bitcast b8 [[LOAD_3_93399]] to i8
+; CHECK-NEXT: [[PTR2_0_93:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 372
+; CHECK-NEXT: [[TMP1214:%.*]] = bitcast i8 [[TMP1210]] to b8
+; CHECK-NEXT: [[TMP1215:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1214]], i32 0
+; CHECK-NEXT: [[TMP1216:%.*]] = bitcast i8 [[TMP1211]] to b8
+; CHECK-NEXT: [[TMP1217:%.*]] = insertelement <4 x b8> [[TMP1215]], b8 [[TMP1216]], i32 1
+; CHECK-NEXT: [[TMP1218:%.*]] = bitcast i8 [[TMP1212]] to b8
+; CHECK-NEXT: [[TMP1219:%.*]] = insertelement <4 x b8> [[TMP1217]], b8 [[TMP1218]], i32 2
+; CHECK-NEXT: [[TMP1220:%.*]] = bitcast i8 [[TMP1213]] to b8
+; CHECK-NEXT: [[TMP1221:%.*]] = insertelement <4 x b8> [[TMP1219]], b8 [[TMP1220]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1221]], ptr [[PTR2_0_93]], align 4
+; CHECK-NEXT: [[PTR_0_94:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 376
+; CHECK-NEXT: [[TMP1222:%.*]] = load <4 x b8>, ptr [[PTR_0_94]], align 4
+; CHECK-NEXT: [[LOAD_0_94400:%.*]] = extractelement <4 x b8> [[TMP1222]], i32 0
+; CHECK-NEXT: [[TMP1223:%.*]] = bitcast b8 [[LOAD_0_94400]] to i8
+; CHECK-NEXT: [[LOAD_1_94401:%.*]] = extractelement <4 x b8> [[TMP1222]], i32 1
+; CHECK-NEXT: [[TMP1224:%.*]] = bitcast b8 [[LOAD_1_94401]] to i8
+; CHECK-NEXT: [[LOAD_2_94402:%.*]] = extractelement <4 x b8> [[TMP1222]], i32 2
+; CHECK-NEXT: [[TMP1225:%.*]] = bitcast b8 [[LOAD_2_94402]] to i8
+; CHECK-NEXT: [[LOAD_3_94403:%.*]] = extractelement <4 x b8> [[TMP1222]], i32 3
+; CHECK-NEXT: [[TMP1226:%.*]] = bitcast b8 [[LOAD_3_94403]] to i8
+; CHECK-NEXT: [[PTR2_0_94:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 376
+; CHECK-NEXT: [[TMP1227:%.*]] = bitcast i8 [[TMP1223]] to b8
+; CHECK-NEXT: [[TMP1228:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1227]], i32 0
+; CHECK-NEXT: [[TMP1229:%.*]] = bitcast i8 [[TMP1224]] to b8
+; CHECK-NEXT: [[TMP1230:%.*]] = insertelement <4 x b8> [[TMP1228]], b8 [[TMP1229]], i32 1
+; CHECK-NEXT: [[TMP1231:%.*]] = bitcast i8 [[TMP1225]] to b8
+; CHECK-NEXT: [[TMP1232:%.*]] = insertelement <4 x b8> [[TMP1230]], b8 [[TMP1231]], i32 2
+; CHECK-NEXT: [[TMP1233:%.*]] = bitcast i8 [[TMP1226]] to b8
+; CHECK-NEXT: [[TMP1234:%.*]] = insertelement <4 x b8> [[TMP1232]], b8 [[TMP1233]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1234]], ptr [[PTR2_0_94]], align 4
+; CHECK-NEXT: [[PTR_0_95:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 380
+; CHECK-NEXT: [[TMP1235:%.*]] = load <4 x b8>, ptr [[PTR_0_95]], align 4
+; CHECK-NEXT: [[LOAD_0_95404:%.*]] = extractelement <4 x b8> [[TMP1235]], i32 0
+; CHECK-NEXT: [[TMP1236:%.*]] = bitcast b8 [[LOAD_0_95404]] to i8
+; CHECK-NEXT: [[LOAD_1_95405:%.*]] = extractelement <4 x b8> [[TMP1235]], i32 1
+; CHECK-NEXT: [[TMP1237:%.*]] = bitcast b8 [[LOAD_1_95405]] to i8
+; CHECK-NEXT: [[LOAD_2_95406:%.*]] = extractelement <4 x b8> [[TMP1235]], i32 2
+; CHECK-NEXT: [[TMP1238:%.*]] = bitcast b8 [[LOAD_2_95406]] to i8
+; CHECK-NEXT: [[LOAD_3_95407:%.*]] = extractelement <4 x b8> [[TMP1235]], i32 3
+; CHECK-NEXT: [[TMP1239:%.*]] = bitcast b8 [[LOAD_3_95407]] to i8
+; CHECK-NEXT: [[PTR2_0_95:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 380
+; CHECK-NEXT: [[TMP1240:%.*]] = bitcast i8 [[TMP1236]] to b8
+; CHECK-NEXT: [[TMP1241:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1240]], i32 0
+; CHECK-NEXT: [[TMP1242:%.*]] = bitcast i8 [[TMP1237]] to b8
+; CHECK-NEXT: [[TMP1243:%.*]] = insertelement <4 x b8> [[TMP1241]], b8 [[TMP1242]], i32 1
+; CHECK-NEXT: [[TMP1244:%.*]] = bitcast i8 [[TMP1238]] to b8
+; CHECK-NEXT: [[TMP1245:%.*]] = insertelement <4 x b8> [[TMP1243]], b8 [[TMP1244]], i32 2
+; CHECK-NEXT: [[TMP1246:%.*]] = bitcast i8 [[TMP1239]] to b8
+; CHECK-NEXT: [[TMP1247:%.*]] = insertelement <4 x b8> [[TMP1245]], b8 [[TMP1246]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1247]], ptr [[PTR2_0_95]], align 4
+; CHECK-NEXT: [[PTR_0_96:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 384
+; CHECK-NEXT: [[TMP1248:%.*]] = load <4 x b8>, ptr [[PTR_0_96]], align 4
+; CHECK-NEXT: [[LOAD_0_96408:%.*]] = extractelement <4 x b8> [[TMP1248]], i32 0
+; CHECK-NEXT: [[TMP1249:%.*]] = bitcast b8 [[LOAD_0_96408]] to i8
+; CHECK-NEXT: [[LOAD_1_96409:%.*]] = extractelement <4 x b8> [[TMP1248]], i32 1
+; CHECK-NEXT: [[TMP1250:%.*]] = bitcast b8 [[LOAD_1_96409]] to i8
+; CHECK-NEXT: [[LOAD_2_96410:%.*]] = extractelement <4 x b8> [[TMP1248]], i32 2
+; CHECK-NEXT: [[TMP1251:%.*]] = bitcast b8 [[LOAD_2_96410]] to i8
+; CHECK-NEXT: [[LOAD_3_96411:%.*]] = extractelement <4 x b8> [[TMP1248]], i32 3
+; CHECK-NEXT: [[TMP1252:%.*]] = bitcast b8 [[LOAD_3_96411]] to i8
+; CHECK-NEXT: [[PTR2_0_96:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 384
+; CHECK-NEXT: [[TMP1253:%.*]] = bitcast i8 [[TMP1249]] to b8
+; CHECK-NEXT: [[TMP1254:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1253]], i32 0
+; CHECK-NEXT: [[TMP1255:%.*]] = bitcast i8 [[TMP1250]] to b8
+; CHECK-NEXT: [[TMP1256:%.*]] = insertelement <4 x b8> [[TMP1254]], b8 [[TMP1255]], i32 1
+; CHECK-NEXT: [[TMP1257:%.*]] = bitcast i8 [[TMP1251]] to b8
+; CHECK-NEXT: [[TMP1258:%.*]] = insertelement <4 x b8> [[TMP1256]], b8 [[TMP1257]], i32 2
+; CHECK-NEXT: [[TMP1259:%.*]] = bitcast i8 [[TMP1252]] to b8
+; CHECK-NEXT: [[TMP1260:%.*]] = insertelement <4 x b8> [[TMP1258]], b8 [[TMP1259]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1260]], ptr [[PTR2_0_96]], align 4
+; CHECK-NEXT: [[PTR_0_97:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 388
+; CHECK-NEXT: [[TMP1261:%.*]] = load <4 x b8>, ptr [[PTR_0_97]], align 4
+; CHECK-NEXT: [[LOAD_0_97412:%.*]] = extractelement <4 x b8> [[TMP1261]], i32 0
+; CHECK-NEXT: [[TMP1262:%.*]] = bitcast b8 [[LOAD_0_97412]] to i8
+; CHECK-NEXT: [[LOAD_1_97413:%.*]] = extractelement <4 x b8> [[TMP1261]], i32 1
+; CHECK-NEXT: [[TMP1263:%.*]] = bitcast b8 [[LOAD_1_97413]] to i8
+; CHECK-NEXT: [[LOAD_2_97414:%.*]] = extractelement <4 x b8> [[TMP1261]], i32 2
+; CHECK-NEXT: [[TMP1264:%.*]] = bitcast b8 [[LOAD_2_97414]] to i8
+; CHECK-NEXT: [[LOAD_3_97415:%.*]] = extractelement <4 x b8> [[TMP1261]], i32 3
+; CHECK-NEXT: [[TMP1265:%.*]] = bitcast b8 [[LOAD_3_97415]] to i8
+; CHECK-NEXT: [[PTR2_0_97:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 388
+; CHECK-NEXT: [[TMP1266:%.*]] = bitcast i8 [[TMP1262]] to b8
+; CHECK-NEXT: [[TMP1267:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1266]], i32 0
+; CHECK-NEXT: [[TMP1268:%.*]] = bitcast i8 [[TMP1263]] to b8
+; CHECK-NEXT: [[TMP1269:%.*]] = insertelement <4 x b8> [[TMP1267]], b8 [[TMP1268]], i32 1
+; CHECK-NEXT: [[TMP1270:%.*]] = bitcast i8 [[TMP1264]] to b8
+; CHECK-NEXT: [[TMP1271:%.*]] = insertelement <4 x b8> [[TMP1269]], b8 [[TMP1270]], i32 2
+; CHECK-NEXT: [[TMP1272:%.*]] = bitcast i8 [[TMP1265]] to b8
+; CHECK-NEXT: [[TMP1273:%.*]] = insertelement <4 x b8> [[TMP1271]], b8 [[TMP1272]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1273]], ptr [[PTR2_0_97]], align 4
+; CHECK-NEXT: [[PTR_0_98:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 392
+; CHECK-NEXT: [[TMP1274:%.*]] = load <4 x b8>, ptr [[PTR_0_98]], align 4
+; CHECK-NEXT: [[LOAD_0_98416:%.*]] = extractelement <4 x b8> [[TMP1274]], i32 0
+; CHECK-NEXT: [[TMP1275:%.*]] = bitcast b8 [[LOAD_0_98416]] to i8
+; CHECK-NEXT: [[LOAD_1_98417:%.*]] = extractelement <4 x b8> [[TMP1274]], i32 1
+; CHECK-NEXT: [[TMP1276:%.*]] = bitcast b8 [[LOAD_1_98417]] to i8
+; CHECK-NEXT: [[LOAD_2_98418:%.*]] = extractelement <4 x b8> [[TMP1274]], i32 2
+; CHECK-NEXT: [[TMP1277:%.*]] = bitcast b8 [[LOAD_2_98418]] to i8
+; CHECK-NEXT: [[LOAD_3_98419:%.*]] = extractelement <4 x b8> [[TMP1274]], i32 3
+; CHECK-NEXT: [[TMP1278:%.*]] = bitcast b8 [[LOAD_3_98419]] to i8
+; CHECK-NEXT: [[PTR2_0_98:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 392
+; CHECK-NEXT: [[TMP1279:%.*]] = bitcast i8 [[TMP1275]] to b8
+; CHECK-NEXT: [[TMP1280:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1279]], i32 0
+; CHECK-NEXT: [[TMP1281:%.*]] = bitcast i8 [[TMP1276]] to b8
+; CHECK-NEXT: [[TMP1282:%.*]] = insertelement <4 x b8> [[TMP1280]], b8 [[TMP1281]], i32 1
+; CHECK-NEXT: [[TMP1283:%.*]] = bitcast i8 [[TMP1277]] to b8
+; CHECK-NEXT: [[TMP1284:%.*]] = insertelement <4 x b8> [[TMP1282]], b8 [[TMP1283]], i32 2
+; CHECK-NEXT: [[TMP1285:%.*]] = bitcast i8 [[TMP1278]] to b8
+; CHECK-NEXT: [[TMP1286:%.*]] = insertelement <4 x b8> [[TMP1284]], b8 [[TMP1285]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1286]], ptr [[PTR2_0_98]], align 4
+; CHECK-NEXT: [[PTR_0_99:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 396
+; CHECK-NEXT: [[TMP1287:%.*]] = load <4 x b8>, ptr [[PTR_0_99]], align 4
+; CHECK-NEXT: [[LOAD_0_99420:%.*]] = extractelement <4 x b8> [[TMP1287]], i32 0
+; CHECK-NEXT: [[TMP1288:%.*]] = bitcast b8 [[LOAD_0_99420]] to i8
+; CHECK-NEXT: [[LOAD_1_99421:%.*]] = extractelement <4 x b8> [[TMP1287]], i32 1
+; CHECK-NEXT: [[TMP1289:%.*]] = bitcast b8 [[LOAD_1_99421]] to i8
+; CHECK-NEXT: [[LOAD_2_99422:%.*]] = extractelement <4 x b8> [[TMP1287]], i32 2
+; CHECK-NEXT: [[TMP1290:%.*]] = bitcast b8 [[LOAD_2_99422]] to i8
+; CHECK-NEXT: [[LOAD_3_99423:%.*]] = extractelement <4 x b8> [[TMP1287]], i32 3
+; CHECK-NEXT: [[TMP1291:%.*]] = bitcast b8 [[LOAD_3_99423]] to i8
+; CHECK-NEXT: [[PTR2_0_99:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 396
+; CHECK-NEXT: [[TMP1292:%.*]] = bitcast i8 [[TMP1288]] to b8
+; CHECK-NEXT: [[TMP1293:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1292]], i32 0
+; CHECK-NEXT: [[TMP1294:%.*]] = bitcast i8 [[TMP1289]] to b8
+; CHECK-NEXT: [[TMP1295:%.*]] = insertelement <4 x b8> [[TMP1293]], b8 [[TMP1294]], i32 1
+; CHECK-NEXT: [[TMP1296:%.*]] = bitcast i8 [[TMP1290]] to b8
+; CHECK-NEXT: [[TMP1297:%.*]] = insertelement <4 x b8> [[TMP1295]], b8 [[TMP1296]], i32 2
+; CHECK-NEXT: [[TMP1298:%.*]] = bitcast i8 [[TMP1291]] to b8
+; CHECK-NEXT: [[TMP1299:%.*]] = insertelement <4 x b8> [[TMP1297]], b8 [[TMP1298]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1299]], ptr [[PTR2_0_99]], align 4
+; CHECK-NEXT: [[PTR_0_100:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 400
+; CHECK-NEXT: [[TMP1300:%.*]] = load <4 x b8>, ptr [[PTR_0_100]], align 4
+; CHECK-NEXT: [[LOAD_0_100424:%.*]] = extractelement <4 x b8> [[TMP1300]], i32 0
+; CHECK-NEXT: [[TMP1301:%.*]] = bitcast b8 [[LOAD_0_100424]] to i8
+; CHECK-NEXT: [[LOAD_1_100425:%.*]] = extractelement <4 x b8> [[TMP1300]], i32 1
+; CHECK-NEXT: [[TMP1302:%.*]] = bitcast b8 [[LOAD_1_100425]] to i8
+; CHECK-NEXT: [[LOAD_2_100426:%.*]] = extractelement <4 x b8> [[TMP1300]], i32 2
+; CHECK-NEXT: [[TMP1303:%.*]] = bitcast b8 [[LOAD_2_100426]] to i8
+; CHECK-NEXT: [[LOAD_3_100427:%.*]] = extractelement <4 x b8> [[TMP1300]], i32 3
+; CHECK-NEXT: [[TMP1304:%.*]] = bitcast b8 [[LOAD_3_100427]] to i8
+; CHECK-NEXT: [[PTR2_0_100:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 400
+; CHECK-NEXT: [[TMP1305:%.*]] = bitcast i8 [[TMP1301]] to b8
+; CHECK-NEXT: [[TMP1306:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1305]], i32 0
+; CHECK-NEXT: [[TMP1307:%.*]] = bitcast i8 [[TMP1302]] to b8
+; CHECK-NEXT: [[TMP1308:%.*]] = insertelement <4 x b8> [[TMP1306]], b8 [[TMP1307]], i32 1
+; CHECK-NEXT: [[TMP1309:%.*]] = bitcast i8 [[TMP1303]] to b8
+; CHECK-NEXT: [[TMP1310:%.*]] = insertelement <4 x b8> [[TMP1308]], b8 [[TMP1309]], i32 2
+; CHECK-NEXT: [[TMP1311:%.*]] = bitcast i8 [[TMP1304]] to b8
+; CHECK-NEXT: [[TMP1312:%.*]] = insertelement <4 x b8> [[TMP1310]], b8 [[TMP1311]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1312]], ptr [[PTR2_0_100]], align 4
+; CHECK-NEXT: [[PTR_0_101:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 404
+; CHECK-NEXT: [[TMP1313:%.*]] = load <4 x b8>, ptr [[PTR_0_101]], align 4
+; CHECK-NEXT: [[LOAD_0_101428:%.*]] = extractelement <4 x b8> [[TMP1313]], i32 0
+; CHECK-NEXT: [[TMP1314:%.*]] = bitcast b8 [[LOAD_0_101428]] to i8
+; CHECK-NEXT: [[LOAD_1_101429:%.*]] = extractelement <4 x b8> [[TMP1313]], i32 1
+; CHECK-NEXT: [[TMP1315:%.*]] = bitcast b8 [[LOAD_1_101429]] to i8
+; CHECK-NEXT: [[LOAD_2_101430:%.*]] = extractelement <4 x b8> [[TMP1313]], i32 2
+; CHECK-NEXT: [[TMP1316:%.*]] = bitcast b8 [[LOAD_2_101430]] to i8
+; CHECK-NEXT: [[LOAD_3_101431:%.*]] = extractelement <4 x b8> [[TMP1313]], i32 3
+; CHECK-NEXT: [[TMP1317:%.*]] = bitcast b8 [[LOAD_3_101431]] to i8
+; CHECK-NEXT: [[PTR2_0_101:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 404
+; CHECK-NEXT: [[TMP1318:%.*]] = bitcast i8 [[TMP1314]] to b8
+; CHECK-NEXT: [[TMP1319:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1318]], i32 0
+; CHECK-NEXT: [[TMP1320:%.*]] = bitcast i8 [[TMP1315]] to b8
+; CHECK-NEXT: [[TMP1321:%.*]] = insertelement <4 x b8> [[TMP1319]], b8 [[TMP1320]], i32 1
+; CHECK-NEXT: [[TMP1322:%.*]] = bitcast i8 [[TMP1316]] to b8
+; CHECK-NEXT: [[TMP1323:%.*]] = insertelement <4 x b8> [[TMP1321]], b8 [[TMP1322]], i32 2
+; CHECK-NEXT: [[TMP1324:%.*]] = bitcast i8 [[TMP1317]] to b8
+; CHECK-NEXT: [[TMP1325:%.*]] = insertelement <4 x b8> [[TMP1323]], b8 [[TMP1324]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1325]], ptr [[PTR2_0_101]], align 4
+; CHECK-NEXT: [[PTR_0_102:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 408
+; CHECK-NEXT: [[TMP1326:%.*]] = load <4 x b8>, ptr [[PTR_0_102]], align 4
+; CHECK-NEXT: [[LOAD_0_102432:%.*]] = extractelement <4 x b8> [[TMP1326]], i32 0
+; CHECK-NEXT: [[TMP1327:%.*]] = bitcast b8 [[LOAD_0_102432]] to i8
+; CHECK-NEXT: [[LOAD_1_102433:%.*]] = extractelement <4 x b8> [[TMP1326]], i32 1
+; CHECK-NEXT: [[TMP1328:%.*]] = bitcast b8 [[LOAD_1_102433]] to i8
+; CHECK-NEXT: [[LOAD_2_102434:%.*]] = extractelement <4 x b8> [[TMP1326]], i32 2
+; CHECK-NEXT: [[TMP1329:%.*]] = bitcast b8 [[LOAD_2_102434]] to i8
+; CHECK-NEXT: [[LOAD_3_102435:%.*]] = extractelement <4 x b8> [[TMP1326]], i32 3
+; CHECK-NEXT: [[TMP1330:%.*]] = bitcast b8 [[LOAD_3_102435]] to i8
+; CHECK-NEXT: [[PTR2_0_102:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 408
+; CHECK-NEXT: [[TMP1331:%.*]] = bitcast i8 [[TMP1327]] to b8
+; CHECK-NEXT: [[TMP1332:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1331]], i32 0
+; CHECK-NEXT: [[TMP1333:%.*]] = bitcast i8 [[TMP1328]] to b8
+; CHECK-NEXT: [[TMP1334:%.*]] = insertelement <4 x b8> [[TMP1332]], b8 [[TMP1333]], i32 1
+; CHECK-NEXT: [[TMP1335:%.*]] = bitcast i8 [[TMP1329]] to b8
+; CHECK-NEXT: [[TMP1336:%.*]] = insertelement <4 x b8> [[TMP1334]], b8 [[TMP1335]], i32 2
+; CHECK-NEXT: [[TMP1337:%.*]] = bitcast i8 [[TMP1330]] to b8
+; CHECK-NEXT: [[TMP1338:%.*]] = insertelement <4 x b8> [[TMP1336]], b8 [[TMP1337]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1338]], ptr [[PTR2_0_102]], align 4
+; CHECK-NEXT: [[PTR_0_103:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 412
+; CHECK-NEXT: [[TMP1339:%.*]] = load <4 x b8>, ptr [[PTR_0_103]], align 4
+; CHECK-NEXT: [[LOAD_0_103436:%.*]] = extractelement <4 x b8> [[TMP1339]], i32 0
+; CHECK-NEXT: [[TMP1340:%.*]] = bitcast b8 [[LOAD_0_103436]] to i8
+; CHECK-NEXT: [[LOAD_1_103437:%.*]] = extractelement <4 x b8> [[TMP1339]], i32 1
+; CHECK-NEXT: [[TMP1341:%.*]] = bitcast b8 [[LOAD_1_103437]] to i8
+; CHECK-NEXT: [[LOAD_2_103438:%.*]] = extractelement <4 x b8> [[TMP1339]], i32 2
+; CHECK-NEXT: [[TMP1342:%.*]] = bitcast b8 [[LOAD_2_103438]] to i8
+; CHECK-NEXT: [[LOAD_3_103439:%.*]] = extractelement <4 x b8> [[TMP1339]], i32 3
+; CHECK-NEXT: [[TMP1343:%.*]] = bitcast b8 [[LOAD_3_103439]] to i8
+; CHECK-NEXT: [[PTR2_0_103:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 412
+; CHECK-NEXT: [[TMP1344:%.*]] = bitcast i8 [[TMP1340]] to b8
+; CHECK-NEXT: [[TMP1345:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1344]], i32 0
+; CHECK-NEXT: [[TMP1346:%.*]] = bitcast i8 [[TMP1341]] to b8
+; CHECK-NEXT: [[TMP1347:%.*]] = insertelement <4 x b8> [[TMP1345]], b8 [[TMP1346]], i32 1
+; CHECK-NEXT: [[TMP1348:%.*]] = bitcast i8 [[TMP1342]] to b8
+; CHECK-NEXT: [[TMP1349:%.*]] = insertelement <4 x b8> [[TMP1347]], b8 [[TMP1348]], i32 2
+; CHECK-NEXT: [[TMP1350:%.*]] = bitcast i8 [[TMP1343]] to b8
+; CHECK-NEXT: [[TMP1351:%.*]] = insertelement <4 x b8> [[TMP1349]], b8 [[TMP1350]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1351]], ptr [[PTR2_0_103]], align 4
+; CHECK-NEXT: [[PTR_0_104:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 416
+; CHECK-NEXT: [[TMP1352:%.*]] = load <4 x b8>, ptr [[PTR_0_104]], align 4
+; CHECK-NEXT: [[LOAD_0_104440:%.*]] = extractelement <4 x b8> [[TMP1352]], i32 0
+; CHECK-NEXT: [[TMP1353:%.*]] = bitcast b8 [[LOAD_0_104440]] to i8
+; CHECK-NEXT: [[LOAD_1_104441:%.*]] = extractelement <4 x b8> [[TMP1352]], i32 1
+; CHECK-NEXT: [[TMP1354:%.*]] = bitcast b8 [[LOAD_1_104441]] to i8
+; CHECK-NEXT: [[LOAD_2_104442:%.*]] = extractelement <4 x b8> [[TMP1352]], i32 2
+; CHECK-NEXT: [[TMP1355:%.*]] = bitcast b8 [[LOAD_2_104442]] to i8
+; CHECK-NEXT: [[LOAD_3_104443:%.*]] = extractelement <4 x b8> [[TMP1352]], i32 3
+; CHECK-NEXT: [[TMP1356:%.*]] = bitcast b8 [[LOAD_3_104443]] to i8
+; CHECK-NEXT: [[PTR2_0_104:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 416
+; CHECK-NEXT: [[TMP1357:%.*]] = bitcast i8 [[TMP1353]] to b8
+; CHECK-NEXT: [[TMP1358:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1357]], i32 0
+; CHECK-NEXT: [[TMP1359:%.*]] = bitcast i8 [[TMP1354]] to b8
+; CHECK-NEXT: [[TMP1360:%.*]] = insertelement <4 x b8> [[TMP1358]], b8 [[TMP1359]], i32 1
+; CHECK-NEXT: [[TMP1361:%.*]] = bitcast i8 [[TMP1355]] to b8
+; CHECK-NEXT: [[TMP1362:%.*]] = insertelement <4 x b8> [[TMP1360]], b8 [[TMP1361]], i32 2
+; CHECK-NEXT: [[TMP1363:%.*]] = bitcast i8 [[TMP1356]] to b8
+; CHECK-NEXT: [[TMP1364:%.*]] = insertelement <4 x b8> [[TMP1362]], b8 [[TMP1363]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1364]], ptr [[PTR2_0_104]], align 4
+; CHECK-NEXT: [[PTR_0_105:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 420
+; CHECK-NEXT: [[TMP1365:%.*]] = load <4 x b8>, ptr [[PTR_0_105]], align 4
+; CHECK-NEXT: [[LOAD_0_105444:%.*]] = extractelement <4 x b8> [[TMP1365]], i32 0
+; CHECK-NEXT: [[TMP1366:%.*]] = bitcast b8 [[LOAD_0_105444]] to i8
+; CHECK-NEXT: [[LOAD_1_105445:%.*]] = extractelement <4 x b8> [[TMP1365]], i32 1
+; CHECK-NEXT: [[TMP1367:%.*]] = bitcast b8 [[LOAD_1_105445]] to i8
+; CHECK-NEXT: [[LOAD_2_105446:%.*]] = extractelement <4 x b8> [[TMP1365]], i32 2
+; CHECK-NEXT: [[TMP1368:%.*]] = bitcast b8 [[LOAD_2_105446]] to i8
+; CHECK-NEXT: [[LOAD_3_105447:%.*]] = extractelement <4 x b8> [[TMP1365]], i32 3
+; CHECK-NEXT: [[TMP1369:%.*]] = bitcast b8 [[LOAD_3_105447]] to i8
+; CHECK-NEXT: [[PTR2_0_105:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 420
+; CHECK-NEXT: [[TMP1370:%.*]] = bitcast i8 [[TMP1366]] to b8
+; CHECK-NEXT: [[TMP1371:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1370]], i32 0
+; CHECK-NEXT: [[TMP1372:%.*]] = bitcast i8 [[TMP1367]] to b8
+; CHECK-NEXT: [[TMP1373:%.*]] = insertelement <4 x b8> [[TMP1371]], b8 [[TMP1372]], i32 1
+; CHECK-NEXT: [[TMP1374:%.*]] = bitcast i8 [[TMP1368]] to b8
+; CHECK-NEXT: [[TMP1375:%.*]] = insertelement <4 x b8> [[TMP1373]], b8 [[TMP1374]], i32 2
+; CHECK-NEXT: [[TMP1376:%.*]] = bitcast i8 [[TMP1369]] to b8
+; CHECK-NEXT: [[TMP1377:%.*]] = insertelement <4 x b8> [[TMP1375]], b8 [[TMP1376]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1377]], ptr [[PTR2_0_105]], align 4
+; CHECK-NEXT: [[PTR_0_106:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 424
+; CHECK-NEXT: [[TMP1378:%.*]] = load <4 x b8>, ptr [[PTR_0_106]], align 4
+; CHECK-NEXT: [[LOAD_0_106448:%.*]] = extractelement <4 x b8> [[TMP1378]], i32 0
+; CHECK-NEXT: [[TMP1379:%.*]] = bitcast b8 [[LOAD_0_106448]] to i8
+; CHECK-NEXT: [[LOAD_1_106449:%.*]] = extractelement <4 x b8> [[TMP1378]], i32 1
+; CHECK-NEXT: [[TMP1380:%.*]] = bitcast b8 [[LOAD_1_106449]] to i8
+; CHECK-NEXT: [[LOAD_2_106450:%.*]] = extractelement <4 x b8> [[TMP1378]], i32 2
+; CHECK-NEXT: [[TMP1381:%.*]] = bitcast b8 [[LOAD_2_106450]] to i8
+; CHECK-NEXT: [[LOAD_3_106451:%.*]] = extractelement <4 x b8> [[TMP1378]], i32 3
+; CHECK-NEXT: [[TMP1382:%.*]] = bitcast b8 [[LOAD_3_106451]] to i8
+; CHECK-NEXT: [[PTR2_0_106:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 424
+; CHECK-NEXT: [[TMP1383:%.*]] = bitcast i8 [[TMP1379]] to b8
+; CHECK-NEXT: [[TMP1384:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1383]], i32 0
+; CHECK-NEXT: [[TMP1385:%.*]] = bitcast i8 [[TMP1380]] to b8
+; CHECK-NEXT: [[TMP1386:%.*]] = insertelement <4 x b8> [[TMP1384]], b8 [[TMP1385]], i32 1
+; CHECK-NEXT: [[TMP1387:%.*]] = bitcast i8 [[TMP1381]] to b8
+; CHECK-NEXT: [[TMP1388:%.*]] = insertelement <4 x b8> [[TMP1386]], b8 [[TMP1387]], i32 2
+; CHECK-NEXT: [[TMP1389:%.*]] = bitcast i8 [[TMP1382]] to b8
+; CHECK-NEXT: [[TMP1390:%.*]] = insertelement <4 x b8> [[TMP1388]], b8 [[TMP1389]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1390]], ptr [[PTR2_0_106]], align 4
+; CHECK-NEXT: [[PTR_0_107:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 428
+; CHECK-NEXT: [[TMP1391:%.*]] = load <4 x b8>, ptr [[PTR_0_107]], align 4
+; CHECK-NEXT: [[LOAD_0_107452:%.*]] = extractelement <4 x b8> [[TMP1391]], i32 0
+; CHECK-NEXT: [[TMP1392:%.*]] = bitcast b8 [[LOAD_0_107452]] to i8
+; CHECK-NEXT: [[LOAD_1_107453:%.*]] = extractelement <4 x b8> [[TMP1391]], i32 1
+; CHECK-NEXT: [[TMP1393:%.*]] = bitcast b8 [[LOAD_1_107453]] to i8
+; CHECK-NEXT: [[LOAD_2_107454:%.*]] = extractelement <4 x b8> [[TMP1391]], i32 2
+; CHECK-NEXT: [[TMP1394:%.*]] = bitcast b8 [[LOAD_2_107454]] to i8
+; CHECK-NEXT: [[LOAD_3_107455:%.*]] = extractelement <4 x b8> [[TMP1391]], i32 3
+; CHECK-NEXT: [[TMP1395:%.*]] = bitcast b8 [[LOAD_3_107455]] to i8
+; CHECK-NEXT: [[PTR2_0_107:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 428
+; CHECK-NEXT: [[TMP1396:%.*]] = bitcast i8 [[TMP1392]] to b8
+; CHECK-NEXT: [[TMP1397:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1396]], i32 0
+; CHECK-NEXT: [[TMP1398:%.*]] = bitcast i8 [[TMP1393]] to b8
+; CHECK-NEXT: [[TMP1399:%.*]] = insertelement <4 x b8> [[TMP1397]], b8 [[TMP1398]], i32 1
+; CHECK-NEXT: [[TMP1400:%.*]] = bitcast i8 [[TMP1394]] to b8
+; CHECK-NEXT: [[TMP1401:%.*]] = insertelement <4 x b8> [[TMP1399]], b8 [[TMP1400]], i32 2
+; CHECK-NEXT: [[TMP1402:%.*]] = bitcast i8 [[TMP1395]] to b8
+; CHECK-NEXT: [[TMP1403:%.*]] = insertelement <4 x b8> [[TMP1401]], b8 [[TMP1402]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1403]], ptr [[PTR2_0_107]], align 4
+; CHECK-NEXT: [[PTR_0_108:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 432
+; CHECK-NEXT: [[TMP1404:%.*]] = load <4 x b8>, ptr [[PTR_0_108]], align 4
+; CHECK-NEXT: [[LOAD_0_108456:%.*]] = extractelement <4 x b8> [[TMP1404]], i32 0
+; CHECK-NEXT: [[TMP1405:%.*]] = bitcast b8 [[LOAD_0_108456]] to i8
+; CHECK-NEXT: [[LOAD_1_108457:%.*]] = extractelement <4 x b8> [[TMP1404]], i32 1
+; CHECK-NEXT: [[TMP1406:%.*]] = bitcast b8 [[LOAD_1_108457]] to i8
+; CHECK-NEXT: [[LOAD_2_108458:%.*]] = extractelement <4 x b8> [[TMP1404]], i32 2
+; CHECK-NEXT: [[TMP1407:%.*]] = bitcast b8 [[LOAD_2_108458]] to i8
+; CHECK-NEXT: [[LOAD_3_108459:%.*]] = extractelement <4 x b8> [[TMP1404]], i32 3
+; CHECK-NEXT: [[TMP1408:%.*]] = bitcast b8 [[LOAD_3_108459]] to i8
+; CHECK-NEXT: [[PTR2_0_108:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 432
+; CHECK-NEXT: [[TMP1409:%.*]] = bitcast i8 [[TMP1405]] to b8
+; CHECK-NEXT: [[TMP1410:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1409]], i32 0
+; CHECK-NEXT: [[TMP1411:%.*]] = bitcast i8 [[TMP1406]] to b8
+; CHECK-NEXT: [[TMP1412:%.*]] = insertelement <4 x b8> [[TMP1410]], b8 [[TMP1411]], i32 1
+; CHECK-NEXT: [[TMP1413:%.*]] = bitcast i8 [[TMP1407]] to b8
+; CHECK-NEXT: [[TMP1414:%.*]] = insertelement <4 x b8> [[TMP1412]], b8 [[TMP1413]], i32 2
+; CHECK-NEXT: [[TMP1415:%.*]] = bitcast i8 [[TMP1408]] to b8
+; CHECK-NEXT: [[TMP1416:%.*]] = insertelement <4 x b8> [[TMP1414]], b8 [[TMP1415]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1416]], ptr [[PTR2_0_108]], align 4
+; CHECK-NEXT: [[PTR_0_109:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 436
+; CHECK-NEXT: [[TMP1417:%.*]] = load <4 x b8>, ptr [[PTR_0_109]], align 4
+; CHECK-NEXT: [[LOAD_0_109460:%.*]] = extractelement <4 x b8> [[TMP1417]], i32 0
+; CHECK-NEXT: [[TMP1418:%.*]] = bitcast b8 [[LOAD_0_109460]] to i8
+; CHECK-NEXT: [[LOAD_1_109461:%.*]] = extractelement <4 x b8> [[TMP1417]], i32 1
+; CHECK-NEXT: [[TMP1419:%.*]] = bitcast b8 [[LOAD_1_109461]] to i8
+; CHECK-NEXT: [[LOAD_2_109462:%.*]] = extractelement <4 x b8> [[TMP1417]], i32 2
+; CHECK-NEXT: [[TMP1420:%.*]] = bitcast b8 [[LOAD_2_109462]] to i8
+; CHECK-NEXT: [[LOAD_3_109463:%.*]] = extractelement <4 x b8> [[TMP1417]], i32 3
+; CHECK-NEXT: [[TMP1421:%.*]] = bitcast b8 [[LOAD_3_109463]] to i8
+; CHECK-NEXT: [[PTR2_0_109:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 436
+; CHECK-NEXT: [[TMP1422:%.*]] = bitcast i8 [[TMP1418]] to b8
+; CHECK-NEXT: [[TMP1423:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1422]], i32 0
+; CHECK-NEXT: [[TMP1424:%.*]] = bitcast i8 [[TMP1419]] to b8
+; CHECK-NEXT: [[TMP1425:%.*]] = insertelement <4 x b8> [[TMP1423]], b8 [[TMP1424]], i32 1
+; CHECK-NEXT: [[TMP1426:%.*]] = bitcast i8 [[TMP1420]] to b8
+; CHECK-NEXT: [[TMP1427:%.*]] = insertelement <4 x b8> [[TMP1425]], b8 [[TMP1426]], i32 2
+; CHECK-NEXT: [[TMP1428:%.*]] = bitcast i8 [[TMP1421]] to b8
+; CHECK-NEXT: [[TMP1429:%.*]] = insertelement <4 x b8> [[TMP1427]], b8 [[TMP1428]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1429]], ptr [[PTR2_0_109]], align 4
+; CHECK-NEXT: [[PTR_0_110:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 440
+; CHECK-NEXT: [[TMP1430:%.*]] = load <4 x b8>, ptr [[PTR_0_110]], align 4
+; CHECK-NEXT: [[LOAD_0_110464:%.*]] = extractelement <4 x b8> [[TMP1430]], i32 0
+; CHECK-NEXT: [[TMP1431:%.*]] = bitcast b8 [[LOAD_0_110464]] to i8
+; CHECK-NEXT: [[LOAD_1_110465:%.*]] = extractelement <4 x b8> [[TMP1430]], i32 1
+; CHECK-NEXT: [[TMP1432:%.*]] = bitcast b8 [[LOAD_1_110465]] to i8
+; CHECK-NEXT: [[LOAD_2_110466:%.*]] = extractelement <4 x b8> [[TMP1430]], i32 2
+; CHECK-NEXT: [[TMP1433:%.*]] = bitcast b8 [[LOAD_2_110466]] to i8
+; CHECK-NEXT: [[LOAD_3_110467:%.*]] = extractelement <4 x b8> [[TMP1430]], i32 3
+; CHECK-NEXT: [[TMP1434:%.*]] = bitcast b8 [[LOAD_3_110467]] to i8
+; CHECK-NEXT: [[PTR2_0_110:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 440
+; CHECK-NEXT: [[TMP1435:%.*]] = bitcast i8 [[TMP1431]] to b8
+; CHECK-NEXT: [[TMP1436:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1435]], i32 0
+; CHECK-NEXT: [[TMP1437:%.*]] = bitcast i8 [[TMP1432]] to b8
+; CHECK-NEXT: [[TMP1438:%.*]] = insertelement <4 x b8> [[TMP1436]], b8 [[TMP1437]], i32 1
+; CHECK-NEXT: [[TMP1439:%.*]] = bitcast i8 [[TMP1433]] to b8
+; CHECK-NEXT: [[TMP1440:%.*]] = insertelement <4 x b8> [[TMP1438]], b8 [[TMP1439]], i32 2
+; CHECK-NEXT: [[TMP1441:%.*]] = bitcast i8 [[TMP1434]] to b8
+; CHECK-NEXT: [[TMP1442:%.*]] = insertelement <4 x b8> [[TMP1440]], b8 [[TMP1441]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1442]], ptr [[PTR2_0_110]], align 4
+; CHECK-NEXT: [[PTR_0_111:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 444
+; CHECK-NEXT: [[TMP1443:%.*]] = load <4 x b8>, ptr [[PTR_0_111]], align 4
+; CHECK-NEXT: [[LOAD_0_111468:%.*]] = extractelement <4 x b8> [[TMP1443]], i32 0
+; CHECK-NEXT: [[TMP1444:%.*]] = bitcast b8 [[LOAD_0_111468]] to i8
+; CHECK-NEXT: [[LOAD_1_111469:%.*]] = extractelement <4 x b8> [[TMP1443]], i32 1
+; CHECK-NEXT: [[TMP1445:%.*]] = bitcast b8 [[LOAD_1_111469]] to i8
+; CHECK-NEXT: [[LOAD_2_111470:%.*]] = extractelement <4 x b8> [[TMP1443]], i32 2
+; CHECK-NEXT: [[TMP1446:%.*]] = bitcast b8 [[LOAD_2_111470]] to i8
+; CHECK-NEXT: [[LOAD_3_111471:%.*]] = extractelement <4 x b8> [[TMP1443]], i32 3
+; CHECK-NEXT: [[TMP1447:%.*]] = bitcast b8 [[LOAD_3_111471]] to i8
+; CHECK-NEXT: [[PTR2_0_111:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 444
+; CHECK-NEXT: [[TMP1448:%.*]] = bitcast i8 [[TMP1444]] to b8
+; CHECK-NEXT: [[TMP1449:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1448]], i32 0
+; CHECK-NEXT: [[TMP1450:%.*]] = bitcast i8 [[TMP1445]] to b8
+; CHECK-NEXT: [[TMP1451:%.*]] = insertelement <4 x b8> [[TMP1449]], b8 [[TMP1450]], i32 1
+; CHECK-NEXT: [[TMP1452:%.*]] = bitcast i8 [[TMP1446]] to b8
+; CHECK-NEXT: [[TMP1453:%.*]] = insertelement <4 x b8> [[TMP1451]], b8 [[TMP1452]], i32 2
+; CHECK-NEXT: [[TMP1454:%.*]] = bitcast i8 [[TMP1447]] to b8
+; CHECK-NEXT: [[TMP1455:%.*]] = insertelement <4 x b8> [[TMP1453]], b8 [[TMP1454]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1455]], ptr [[PTR2_0_111]], align 4
+; CHECK-NEXT: [[PTR_0_112:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 448
+; CHECK-NEXT: [[TMP1456:%.*]] = load <4 x b8>, ptr [[PTR_0_112]], align 4
+; CHECK-NEXT: [[LOAD_0_112472:%.*]] = extractelement <4 x b8> [[TMP1456]], i32 0
+; CHECK-NEXT: [[TMP1457:%.*]] = bitcast b8 [[LOAD_0_112472]] to i8
+; CHECK-NEXT: [[LOAD_1_112473:%.*]] = extractelement <4 x b8> [[TMP1456]], i32 1
+; CHECK-NEXT: [[TMP1458:%.*]] = bitcast b8 [[LOAD_1_112473]] to i8
+; CHECK-NEXT: [[LOAD_2_112474:%.*]] = extractelement <4 x b8> [[TMP1456]], i32 2
+; CHECK-NEXT: [[TMP1459:%.*]] = bitcast b8 [[LOAD_2_112474]] to i8
+; CHECK-NEXT: [[LOAD_3_112475:%.*]] = extractelement <4 x b8> [[TMP1456]], i32 3
+; CHECK-NEXT: [[TMP1460:%.*]] = bitcast b8 [[LOAD_3_112475]] to i8
+; CHECK-NEXT: [[PTR2_0_112:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 448
+; CHECK-NEXT: [[TMP1461:%.*]] = bitcast i8 [[TMP1457]] to b8
+; CHECK-NEXT: [[TMP1462:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1461]], i32 0
+; CHECK-NEXT: [[TMP1463:%.*]] = bitcast i8 [[TMP1458]] to b8
+; CHECK-NEXT: [[TMP1464:%.*]] = insertelement <4 x b8> [[TMP1462]], b8 [[TMP1463]], i32 1
+; CHECK-NEXT: [[TMP1465:%.*]] = bitcast i8 [[TMP1459]] to b8
+; CHECK-NEXT: [[TMP1466:%.*]] = insertelement <4 x b8> [[TMP1464]], b8 [[TMP1465]], i32 2
+; CHECK-NEXT: [[TMP1467:%.*]] = bitcast i8 [[TMP1460]] to b8
+; CHECK-NEXT: [[TMP1468:%.*]] = insertelement <4 x b8> [[TMP1466]], b8 [[TMP1467]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1468]], ptr [[PTR2_0_112]], align 4
+; CHECK-NEXT: [[PTR_0_113:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 452
+; CHECK-NEXT: [[TMP1469:%.*]] = load <4 x b8>, ptr [[PTR_0_113]], align 4
+; CHECK-NEXT: [[LOAD_0_113476:%.*]] = extractelement <4 x b8> [[TMP1469]], i32 0
+; CHECK-NEXT: [[TMP1470:%.*]] = bitcast b8 [[LOAD_0_113476]] to i8
+; CHECK-NEXT: [[LOAD_1_113477:%.*]] = extractelement <4 x b8> [[TMP1469]], i32 1
+; CHECK-NEXT: [[TMP1471:%.*]] = bitcast b8 [[LOAD_1_113477]] to i8
+; CHECK-NEXT: [[LOAD_2_113478:%.*]] = extractelement <4 x b8> [[TMP1469]], i32 2
+; CHECK-NEXT: [[TMP1472:%.*]] = bitcast b8 [[LOAD_2_113478]] to i8
+; CHECK-NEXT: [[LOAD_3_113479:%.*]] = extractelement <4 x b8> [[TMP1469]], i32 3
+; CHECK-NEXT: [[TMP1473:%.*]] = bitcast b8 [[LOAD_3_113479]] to i8
+; CHECK-NEXT: [[PTR2_0_113:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 452
+; CHECK-NEXT: [[TMP1474:%.*]] = bitcast i8 [[TMP1470]] to b8
+; CHECK-NEXT: [[TMP1475:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1474]], i32 0
+; CHECK-NEXT: [[TMP1476:%.*]] = bitcast i8 [[TMP1471]] to b8
+; CHECK-NEXT: [[TMP1477:%.*]] = insertelement <4 x b8> [[TMP1475]], b8 [[TMP1476]], i32 1
+; CHECK-NEXT: [[TMP1478:%.*]] = bitcast i8 [[TMP1472]] to b8
+; CHECK-NEXT: [[TMP1479:%.*]] = insertelement <4 x b8> [[TMP1477]], b8 [[TMP1478]], i32 2
+; CHECK-NEXT: [[TMP1480:%.*]] = bitcast i8 [[TMP1473]] to b8
+; CHECK-NEXT: [[TMP1481:%.*]] = insertelement <4 x b8> [[TMP1479]], b8 [[TMP1480]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1481]], ptr [[PTR2_0_113]], align 4
+; CHECK-NEXT: [[PTR_0_114:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 456
+; CHECK-NEXT: [[TMP1482:%.*]] = load <4 x b8>, ptr [[PTR_0_114]], align 4
+; CHECK-NEXT: [[LOAD_0_114480:%.*]] = extractelement <4 x b8> [[TMP1482]], i32 0
+; CHECK-NEXT: [[TMP1483:%.*]] = bitcast b8 [[LOAD_0_114480]] to i8
+; CHECK-NEXT: [[LOAD_1_114481:%.*]] = extractelement <4 x b8> [[TMP1482]], i32 1
+; CHECK-NEXT: [[TMP1484:%.*]] = bitcast b8 [[LOAD_1_114481]] to i8
+; CHECK-NEXT: [[LOAD_2_114482:%.*]] = extractelement <4 x b8> [[TMP1482]], i32 2
+; CHECK-NEXT: [[TMP1485:%.*]] = bitcast b8 [[LOAD_2_114482]] to i8
+; CHECK-NEXT: [[LOAD_3_114483:%.*]] = extractelement <4 x b8> [[TMP1482]], i32 3
+; CHECK-NEXT: [[TMP1486:%.*]] = bitcast b8 [[LOAD_3_114483]] to i8
+; CHECK-NEXT: [[PTR2_0_114:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 456
+; CHECK-NEXT: [[TMP1487:%.*]] = bitcast i8 [[TMP1483]] to b8
+; CHECK-NEXT: [[TMP1488:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1487]], i32 0
+; CHECK-NEXT: [[TMP1489:%.*]] = bitcast i8 [[TMP1484]] to b8
+; CHECK-NEXT: [[TMP1490:%.*]] = insertelement <4 x b8> [[TMP1488]], b8 [[TMP1489]], i32 1
+; CHECK-NEXT: [[TMP1491:%.*]] = bitcast i8 [[TMP1485]] to b8
+; CHECK-NEXT: [[TMP1492:%.*]] = insertelement <4 x b8> [[TMP1490]], b8 [[TMP1491]], i32 2
+; CHECK-NEXT: [[TMP1493:%.*]] = bitcast i8 [[TMP1486]] to b8
+; CHECK-NEXT: [[TMP1494:%.*]] = insertelement <4 x b8> [[TMP1492]], b8 [[TMP1493]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1494]], ptr [[PTR2_0_114]], align 4
+; CHECK-NEXT: [[PTR_0_115:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 460
+; CHECK-NEXT: [[TMP1495:%.*]] = load <4 x b8>, ptr [[PTR_0_115]], align 4
+; CHECK-NEXT: [[LOAD_0_115484:%.*]] = extractelement <4 x b8> [[TMP1495]], i32 0
+; CHECK-NEXT: [[TMP1496:%.*]] = bitcast b8 [[LOAD_0_115484]] to i8
+; CHECK-NEXT: [[LOAD_1_115485:%.*]] = extractelement <4 x b8> [[TMP1495]], i32 1
+; CHECK-NEXT: [[TMP1497:%.*]] = bitcast b8 [[LOAD_1_115485]] to i8
+; CHECK-NEXT: [[LOAD_2_115486:%.*]] = extractelement <4 x b8> [[TMP1495]], i32 2
+; CHECK-NEXT: [[TMP1498:%.*]] = bitcast b8 [[LOAD_2_115486]] to i8
+; CHECK-NEXT: [[LOAD_3_115487:%.*]] = extractelement <4 x b8> [[TMP1495]], i32 3
+; CHECK-NEXT: [[TMP1499:%.*]] = bitcast b8 [[LOAD_3_115487]] to i8
+; CHECK-NEXT: [[PTR2_0_115:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 460
+; CHECK-NEXT: [[TMP1500:%.*]] = bitcast i8 [[TMP1496]] to b8
+; CHECK-NEXT: [[TMP1501:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1500]], i32 0
+; CHECK-NEXT: [[TMP1502:%.*]] = bitcast i8 [[TMP1497]] to b8
+; CHECK-NEXT: [[TMP1503:%.*]] = insertelement <4 x b8> [[TMP1501]], b8 [[TMP1502]], i32 1
+; CHECK-NEXT: [[TMP1504:%.*]] = bitcast i8 [[TMP1498]] to b8
+; CHECK-NEXT: [[TMP1505:%.*]] = insertelement <4 x b8> [[TMP1503]], b8 [[TMP1504]], i32 2
+; CHECK-NEXT: [[TMP1506:%.*]] = bitcast i8 [[TMP1499]] to b8
+; CHECK-NEXT: [[TMP1507:%.*]] = insertelement <4 x b8> [[TMP1505]], b8 [[TMP1506]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1507]], ptr [[PTR2_0_115]], align 4
+; CHECK-NEXT: [[PTR_0_116:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 464
+; CHECK-NEXT: [[TMP1508:%.*]] = load <4 x b8>, ptr [[PTR_0_116]], align 4
+; CHECK-NEXT: [[LOAD_0_116488:%.*]] = extractelement <4 x b8> [[TMP1508]], i32 0
+; CHECK-NEXT: [[TMP1509:%.*]] = bitcast b8 [[LOAD_0_116488]] to i8
+; CHECK-NEXT: [[LOAD_1_116489:%.*]] = extractelement <4 x b8> [[TMP1508]], i32 1
+; CHECK-NEXT: [[TMP1510:%.*]] = bitcast b8 [[LOAD_1_116489]] to i8
+; CHECK-NEXT: [[LOAD_2_116490:%.*]] = extractelement <4 x b8> [[TMP1508]], i32 2
+; CHECK-NEXT: [[TMP1511:%.*]] = bitcast b8 [[LOAD_2_116490]] to i8
+; CHECK-NEXT: [[LOAD_3_116491:%.*]] = extractelement <4 x b8> [[TMP1508]], i32 3
+; CHECK-NEXT: [[TMP1512:%.*]] = bitcast b8 [[LOAD_3_116491]] to i8
+; CHECK-NEXT: [[PTR2_0_116:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 464
+; CHECK-NEXT: [[TMP1513:%.*]] = bitcast i8 [[TMP1509]] to b8
+; CHECK-NEXT: [[TMP1514:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1513]], i32 0
+; CHECK-NEXT: [[TMP1515:%.*]] = bitcast i8 [[TMP1510]] to b8
+; CHECK-NEXT: [[TMP1516:%.*]] = insertelement <4 x b8> [[TMP1514]], b8 [[TMP1515]], i32 1
+; CHECK-NEXT: [[TMP1517:%.*]] = bitcast i8 [[TMP1511]] to b8
+; CHECK-NEXT: [[TMP1518:%.*]] = insertelement <4 x b8> [[TMP1516]], b8 [[TMP1517]], i32 2
+; CHECK-NEXT: [[TMP1519:%.*]] = bitcast i8 [[TMP1512]] to b8
+; CHECK-NEXT: [[TMP1520:%.*]] = insertelement <4 x b8> [[TMP1518]], b8 [[TMP1519]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1520]], ptr [[PTR2_0_116]], align 4
+; CHECK-NEXT: [[PTR_0_117:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 468
+; CHECK-NEXT: [[TMP1521:%.*]] = load <4 x b8>, ptr [[PTR_0_117]], align 4
+; CHECK-NEXT: [[LOAD_0_117492:%.*]] = extractelement <4 x b8> [[TMP1521]], i32 0
+; CHECK-NEXT: [[TMP1522:%.*]] = bitcast b8 [[LOAD_0_117492]] to i8
+; CHECK-NEXT: [[LOAD_1_117493:%.*]] = extractelement <4 x b8> [[TMP1521]], i32 1
+; CHECK-NEXT: [[TMP1523:%.*]] = bitcast b8 [[LOAD_1_117493]] to i8
+; CHECK-NEXT: [[LOAD_2_117494:%.*]] = extractelement <4 x b8> [[TMP1521]], i32 2
+; CHECK-NEXT: [[TMP1524:%.*]] = bitcast b8 [[LOAD_2_117494]] to i8
+; CHECK-NEXT: [[LOAD_3_117495:%.*]] = extractelement <4 x b8> [[TMP1521]], i32 3
+; CHECK-NEXT: [[TMP1525:%.*]] = bitcast b8 [[LOAD_3_117495]] to i8
+; CHECK-NEXT: [[PTR2_0_117:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 468
+; CHECK-NEXT: [[TMP1526:%.*]] = bitcast i8 [[TMP1522]] to b8
+; CHECK-NEXT: [[TMP1527:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1526]], i32 0
+; CHECK-NEXT: [[TMP1528:%.*]] = bitcast i8 [[TMP1523]] to b8
+; CHECK-NEXT: [[TMP1529:%.*]] = insertelement <4 x b8> [[TMP1527]], b8 [[TMP1528]], i32 1
+; CHECK-NEXT: [[TMP1530:%.*]] = bitcast i8 [[TMP1524]] to b8
+; CHECK-NEXT: [[TMP1531:%.*]] = insertelement <4 x b8> [[TMP1529]], b8 [[TMP1530]], i32 2
+; CHECK-NEXT: [[TMP1532:%.*]] = bitcast i8 [[TMP1525]] to b8
+; CHECK-NEXT: [[TMP1533:%.*]] = insertelement <4 x b8> [[TMP1531]], b8 [[TMP1532]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1533]], ptr [[PTR2_0_117]], align 4
+; CHECK-NEXT: [[PTR_0_118:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 472
+; CHECK-NEXT: [[TMP1534:%.*]] = load <4 x b8>, ptr [[PTR_0_118]], align 4
+; CHECK-NEXT: [[LOAD_0_118496:%.*]] = extractelement <4 x b8> [[TMP1534]], i32 0
+; CHECK-NEXT: [[TMP1535:%.*]] = bitcast b8 [[LOAD_0_118496]] to i8
+; CHECK-NEXT: [[LOAD_1_118497:%.*]] = extractelement <4 x b8> [[TMP1534]], i32 1
+; CHECK-NEXT: [[TMP1536:%.*]] = bitcast b8 [[LOAD_1_118497]] to i8
+; CHECK-NEXT: [[LOAD_2_118498:%.*]] = extractelement <4 x b8> [[TMP1534]], i32 2
+; CHECK-NEXT: [[TMP1537:%.*]] = bitcast b8 [[LOAD_2_118498]] to i8
+; CHECK-NEXT: [[LOAD_3_118499:%.*]] = extractelement <4 x b8> [[TMP1534]], i32 3
+; CHECK-NEXT: [[TMP1538:%.*]] = bitcast b8 [[LOAD_3_118499]] to i8
+; CHECK-NEXT: [[PTR2_0_118:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 472
+; CHECK-NEXT: [[TMP1539:%.*]] = bitcast i8 [[TMP1535]] to b8
+; CHECK-NEXT: [[TMP1540:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1539]], i32 0
+; CHECK-NEXT: [[TMP1541:%.*]] = bitcast i8 [[TMP1536]] to b8
+; CHECK-NEXT: [[TMP1542:%.*]] = insertelement <4 x b8> [[TMP1540]], b8 [[TMP1541]], i32 1
+; CHECK-NEXT: [[TMP1543:%.*]] = bitcast i8 [[TMP1537]] to b8
+; CHECK-NEXT: [[TMP1544:%.*]] = insertelement <4 x b8> [[TMP1542]], b8 [[TMP1543]], i32 2
+; CHECK-NEXT: [[TMP1545:%.*]] = bitcast i8 [[TMP1538]] to b8
+; CHECK-NEXT: [[TMP1546:%.*]] = insertelement <4 x b8> [[TMP1544]], b8 [[TMP1545]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1546]], ptr [[PTR2_0_118]], align 4
+; CHECK-NEXT: [[PTR_0_119:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 476
+; CHECK-NEXT: [[TMP1547:%.*]] = load <4 x b8>, ptr [[PTR_0_119]], align 4
+; CHECK-NEXT: [[LOAD_0_119500:%.*]] = extractelement <4 x b8> [[TMP1547]], i32 0
+; CHECK-NEXT: [[TMP1548:%.*]] = bitcast b8 [[LOAD_0_119500]] to i8
+; CHECK-NEXT: [[LOAD_1_119501:%.*]] = extractelement <4 x b8> [[TMP1547]], i32 1
+; CHECK-NEXT: [[TMP1549:%.*]] = bitcast b8 [[LOAD_1_119501]] to i8
+; CHECK-NEXT: [[LOAD_2_119502:%.*]] = extractelement <4 x b8> [[TMP1547]], i32 2
+; CHECK-NEXT: [[TMP1550:%.*]] = bitcast b8 [[LOAD_2_119502]] to i8
+; CHECK-NEXT: [[LOAD_3_119503:%.*]] = extractelement <4 x b8> [[TMP1547]], i32 3
+; CHECK-NEXT: [[TMP1551:%.*]] = bitcast b8 [[LOAD_3_119503]] to i8
+; CHECK-NEXT: [[PTR2_0_119:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 476
+; CHECK-NEXT: [[TMP1552:%.*]] = bitcast i8 [[TMP1548]] to b8
+; CHECK-NEXT: [[TMP1553:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1552]], i32 0
+; CHECK-NEXT: [[TMP1554:%.*]] = bitcast i8 [[TMP1549]] to b8
+; CHECK-NEXT: [[TMP1555:%.*]] = insertelement <4 x b8> [[TMP1553]], b8 [[TMP1554]], i32 1
+; CHECK-NEXT: [[TMP1556:%.*]] = bitcast i8 [[TMP1550]] to b8
+; CHECK-NEXT: [[TMP1557:%.*]] = insertelement <4 x b8> [[TMP1555]], b8 [[TMP1556]], i32 2
+; CHECK-NEXT: [[TMP1558:%.*]] = bitcast i8 [[TMP1551]] to b8
+; CHECK-NEXT: [[TMP1559:%.*]] = insertelement <4 x b8> [[TMP1557]], b8 [[TMP1558]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1559]], ptr [[PTR2_0_119]], align 4
+; CHECK-NEXT: [[PTR_0_120:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 480
+; CHECK-NEXT: [[TMP1560:%.*]] = load <4 x b8>, ptr [[PTR_0_120]], align 4
+; CHECK-NEXT: [[LOAD_0_120504:%.*]] = extractelement <4 x b8> [[TMP1560]], i32 0
+; CHECK-NEXT: [[TMP1561:%.*]] = bitcast b8 [[LOAD_0_120504]] to i8
+; CHECK-NEXT: [[LOAD_1_120505:%.*]] = extractelement <4 x b8> [[TMP1560]], i32 1
+; CHECK-NEXT: [[TMP1562:%.*]] = bitcast b8 [[LOAD_1_120505]] to i8
+; CHECK-NEXT: [[LOAD_2_120506:%.*]] = extractelement <4 x b8> [[TMP1560]], i32 2
+; CHECK-NEXT: [[TMP1563:%.*]] = bitcast b8 [[LOAD_2_120506]] to i8
+; CHECK-NEXT: [[LOAD_3_120507:%.*]] = extractelement <4 x b8> [[TMP1560]], i32 3
+; CHECK-NEXT: [[TMP1564:%.*]] = bitcast b8 [[LOAD_3_120507]] to i8
+; CHECK-NEXT: [[PTR2_0_120:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 480
+; CHECK-NEXT: [[TMP1565:%.*]] = bitcast i8 [[TMP1561]] to b8
+; CHECK-NEXT: [[TMP1566:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1565]], i32 0
+; CHECK-NEXT: [[TMP1567:%.*]] = bitcast i8 [[TMP1562]] to b8
+; CHECK-NEXT: [[TMP1568:%.*]] = insertelement <4 x b8> [[TMP1566]], b8 [[TMP1567]], i32 1
+; CHECK-NEXT: [[TMP1569:%.*]] = bitcast i8 [[TMP1563]] to b8
+; CHECK-NEXT: [[TMP1570:%.*]] = insertelement <4 x b8> [[TMP1568]], b8 [[TMP1569]], i32 2
+; CHECK-NEXT: [[TMP1571:%.*]] = bitcast i8 [[TMP1564]] to b8
+; CHECK-NEXT: [[TMP1572:%.*]] = insertelement <4 x b8> [[TMP1570]], b8 [[TMP1571]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1572]], ptr [[PTR2_0_120]], align 4
+; CHECK-NEXT: [[PTR_0_121:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 484
+; CHECK-NEXT: [[TMP1573:%.*]] = load <4 x b8>, ptr [[PTR_0_121]], align 4
+; CHECK-NEXT: [[LOAD_0_121508:%.*]] = extractelement <4 x b8> [[TMP1573]], i32 0
+; CHECK-NEXT: [[TMP1574:%.*]] = bitcast b8 [[LOAD_0_121508]] to i8
+; CHECK-NEXT: [[LOAD_1_121509:%.*]] = extractelement <4 x b8> [[TMP1573]], i32 1
+; CHECK-NEXT: [[TMP1575:%.*]] = bitcast b8 [[LOAD_1_121509]] to i8
+; CHECK-NEXT: [[LOAD_2_121510:%.*]] = extractelement <4 x b8> [[TMP1573]], i32 2
+; CHECK-NEXT: [[TMP1576:%.*]] = bitcast b8 [[LOAD_2_121510]] to i8
+; CHECK-NEXT: [[LOAD_3_121511:%.*]] = extractelement <4 x b8> [[TMP1573]], i32 3
+; CHECK-NEXT: [[TMP1577:%.*]] = bitcast b8 [[LOAD_3_121511]] to i8
+; CHECK-NEXT: [[PTR2_0_121:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 484
+; CHECK-NEXT: [[TMP1578:%.*]] = bitcast i8 [[TMP1574]] to b8
+; CHECK-NEXT: [[TMP1579:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1578]], i32 0
+; CHECK-NEXT: [[TMP1580:%.*]] = bitcast i8 [[TMP1575]] to b8
+; CHECK-NEXT: [[TMP1581:%.*]] = insertelement <4 x b8> [[TMP1579]], b8 [[TMP1580]], i32 1
+; CHECK-NEXT: [[TMP1582:%.*]] = bitcast i8 [[TMP1576]] to b8
+; CHECK-NEXT: [[TMP1583:%.*]] = insertelement <4 x b8> [[TMP1581]], b8 [[TMP1582]], i32 2
+; CHECK-NEXT: [[TMP1584:%.*]] = bitcast i8 [[TMP1577]] to b8
+; CHECK-NEXT: [[TMP1585:%.*]] = insertelement <4 x b8> [[TMP1583]], b8 [[TMP1584]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1585]], ptr [[PTR2_0_121]], align 4
+; CHECK-NEXT: [[PTR_0_122:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 488
+; CHECK-NEXT: [[TMP1586:%.*]] = load <4 x b8>, ptr [[PTR_0_122]], align 4
+; CHECK-NEXT: [[LOAD_0_122512:%.*]] = extractelement <4 x b8> [[TMP1586]], i32 0
+; CHECK-NEXT: [[TMP1587:%.*]] = bitcast b8 [[LOAD_0_122512]] to i8
+; CHECK-NEXT: [[LOAD_1_122513:%.*]] = extractelement <4 x b8> [[TMP1586]], i32 1
+; CHECK-NEXT: [[TMP1588:%.*]] = bitcast b8 [[LOAD_1_122513]] to i8
+; CHECK-NEXT: [[LOAD_2_122514:%.*]] = extractelement <4 x b8> [[TMP1586]], i32 2
+; CHECK-NEXT: [[TMP1589:%.*]] = bitcast b8 [[LOAD_2_122514]] to i8
+; CHECK-NEXT: [[LOAD_3_122515:%.*]] = extractelement <4 x b8> [[TMP1586]], i32 3
+; CHECK-NEXT: [[TMP1590:%.*]] = bitcast b8 [[LOAD_3_122515]] to i8
+; CHECK-NEXT: [[PTR2_0_122:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 488
+; CHECK-NEXT: [[TMP1591:%.*]] = bitcast i8 [[TMP1587]] to b8
+; CHECK-NEXT: [[TMP1592:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1591]], i32 0
+; CHECK-NEXT: [[TMP1593:%.*]] = bitcast i8 [[TMP1588]] to b8
+; CHECK-NEXT: [[TMP1594:%.*]] = insertelement <4 x b8> [[TMP1592]], b8 [[TMP1593]], i32 1
+; CHECK-NEXT: [[TMP1595:%.*]] = bitcast i8 [[TMP1589]] to b8
+; CHECK-NEXT: [[TMP1596:%.*]] = insertelement <4 x b8> [[TMP1594]], b8 [[TMP1595]], i32 2
+; CHECK-NEXT: [[TMP1597:%.*]] = bitcast i8 [[TMP1590]] to b8
+; CHECK-NEXT: [[TMP1598:%.*]] = insertelement <4 x b8> [[TMP1596]], b8 [[TMP1597]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1598]], ptr [[PTR2_0_122]], align 4
+; CHECK-NEXT: [[PTR_0_123:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 492
+; CHECK-NEXT: [[TMP1599:%.*]] = load <4 x b8>, ptr [[PTR_0_123]], align 4
+; CHECK-NEXT: [[LOAD_0_123516:%.*]] = extractelement <4 x b8> [[TMP1599]], i32 0
+; CHECK-NEXT: [[TMP1600:%.*]] = bitcast b8 [[LOAD_0_123516]] to i8
+; CHECK-NEXT: [[LOAD_1_123517:%.*]] = extractelement <4 x b8> [[TMP1599]], i32 1
+; CHECK-NEXT: [[TMP1601:%.*]] = bitcast b8 [[LOAD_1_123517]] to i8
+; CHECK-NEXT: [[LOAD_2_123518:%.*]] = extractelement <4 x b8> [[TMP1599]], i32 2
+; CHECK-NEXT: [[TMP1602:%.*]] = bitcast b8 [[LOAD_2_123518]] to i8
+; CHECK-NEXT: [[LOAD_3_123519:%.*]] = extractelement <4 x b8> [[TMP1599]], i32 3
+; CHECK-NEXT: [[TMP1603:%.*]] = bitcast b8 [[LOAD_3_123519]] to i8
+; CHECK-NEXT: [[PTR2_0_123:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 492
+; CHECK-NEXT: [[TMP1604:%.*]] = bitcast i8 [[TMP1600]] to b8
+; CHECK-NEXT: [[TMP1605:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1604]], i32 0
+; CHECK-NEXT: [[TMP1606:%.*]] = bitcast i8 [[TMP1601]] to b8
+; CHECK-NEXT: [[TMP1607:%.*]] = insertelement <4 x b8> [[TMP1605]], b8 [[TMP1606]], i32 1
+; CHECK-NEXT: [[TMP1608:%.*]] = bitcast i8 [[TMP1602]] to b8
+; CHECK-NEXT: [[TMP1609:%.*]] = insertelement <4 x b8> [[TMP1607]], b8 [[TMP1608]], i32 2
+; CHECK-NEXT: [[TMP1610:%.*]] = bitcast i8 [[TMP1603]] to b8
+; CHECK-NEXT: [[TMP1611:%.*]] = insertelement <4 x b8> [[TMP1609]], b8 [[TMP1610]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1611]], ptr [[PTR2_0_123]], align 4
+; CHECK-NEXT: [[PTR_0_124:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 496
+; CHECK-NEXT: [[TMP1612:%.*]] = load <4 x b8>, ptr [[PTR_0_124]], align 4
+; CHECK-NEXT: [[LOAD_0_124520:%.*]] = extractelement <4 x b8> [[TMP1612]], i32 0
+; CHECK-NEXT: [[TMP1613:%.*]] = bitcast b8 [[LOAD_0_124520]] to i8
+; CHECK-NEXT: [[LOAD_1_124521:%.*]] = extractelement <4 x b8> [[TMP1612]], i32 1
+; CHECK-NEXT: [[TMP1614:%.*]] = bitcast b8 [[LOAD_1_124521]] to i8
+; CHECK-NEXT: [[LOAD_2_124522:%.*]] = extractelement <4 x b8> [[TMP1612]], i32 2
+; CHECK-NEXT: [[TMP1615:%.*]] = bitcast b8 [[LOAD_2_124522]] to i8
+; CHECK-NEXT: [[LOAD_3_124523:%.*]] = extractelement <4 x b8> [[TMP1612]], i32 3
+; CHECK-NEXT: [[TMP1616:%.*]] = bitcast b8 [[LOAD_3_124523]] to i8
+; CHECK-NEXT: [[PTR2_0_124:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 496
+; CHECK-NEXT: [[TMP1617:%.*]] = bitcast i8 [[TMP1613]] to b8
+; CHECK-NEXT: [[TMP1618:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1617]], i32 0
+; CHECK-NEXT: [[TMP1619:%.*]] = bitcast i8 [[TMP1614]] to b8
+; CHECK-NEXT: [[TMP1620:%.*]] = insertelement <4 x b8> [[TMP1618]], b8 [[TMP1619]], i32 1
+; CHECK-NEXT: [[TMP1621:%.*]] = bitcast i8 [[TMP1615]] to b8
+; CHECK-NEXT: [[TMP1622:%.*]] = insertelement <4 x b8> [[TMP1620]], b8 [[TMP1621]], i32 2
+; CHECK-NEXT: [[TMP1623:%.*]] = bitcast i8 [[TMP1616]] to b8
+; CHECK-NEXT: [[TMP1624:%.*]] = insertelement <4 x b8> [[TMP1622]], b8 [[TMP1623]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1624]], ptr [[PTR2_0_124]], align 4
+; CHECK-NEXT: [[PTR_0_125:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 500
+; CHECK-NEXT: [[TMP1625:%.*]] = load <4 x b8>, ptr [[PTR_0_125]], align 4
+; CHECK-NEXT: [[LOAD_0_125524:%.*]] = extractelement <4 x b8> [[TMP1625]], i32 0
+; CHECK-NEXT: [[TMP1626:%.*]] = bitcast b8 [[LOAD_0_125524]] to i8
+; CHECK-NEXT: [[LOAD_1_125525:%.*]] = extractelement <4 x b8> [[TMP1625]], i32 1
+; CHECK-NEXT: [[TMP1627:%.*]] = bitcast b8 [[LOAD_1_125525]] to i8
+; CHECK-NEXT: [[LOAD_2_125526:%.*]] = extractelement <4 x b8> [[TMP1625]], i32 2
+; CHECK-NEXT: [[TMP1628:%.*]] = bitcast b8 [[LOAD_2_125526]] to i8
+; CHECK-NEXT: [[LOAD_3_125527:%.*]] = extractelement <4 x b8> [[TMP1625]], i32 3
+; CHECK-NEXT: [[TMP1629:%.*]] = bitcast b8 [[LOAD_3_125527]] to i8
+; CHECK-NEXT: [[PTR2_0_125:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 500
+; CHECK-NEXT: [[TMP1630:%.*]] = bitcast i8 [[TMP1626]] to b8
+; CHECK-NEXT: [[TMP1631:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1630]], i32 0
+; CHECK-NEXT: [[TMP1632:%.*]] = bitcast i8 [[TMP1627]] to b8
+; CHECK-NEXT: [[TMP1633:%.*]] = insertelement <4 x b8> [[TMP1631]], b8 [[TMP1632]], i32 1
+; CHECK-NEXT: [[TMP1634:%.*]] = bitcast i8 [[TMP1628]] to b8
+; CHECK-NEXT: [[TMP1635:%.*]] = insertelement <4 x b8> [[TMP1633]], b8 [[TMP1634]], i32 2
+; CHECK-NEXT: [[TMP1636:%.*]] = bitcast i8 [[TMP1629]] to b8
+; CHECK-NEXT: [[TMP1637:%.*]] = insertelement <4 x b8> [[TMP1635]], b8 [[TMP1636]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1637]], ptr [[PTR2_0_125]], align 4
+; CHECK-NEXT: [[PTR_0_126:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 504
+; CHECK-NEXT: [[TMP1638:%.*]] = load <4 x b8>, ptr [[PTR_0_126]], align 4
+; CHECK-NEXT: [[LOAD_0_126528:%.*]] = extractelement <4 x b8> [[TMP1638]], i32 0
+; CHECK-NEXT: [[TMP1639:%.*]] = bitcast b8 [[LOAD_0_126528]] to i8
+; CHECK-NEXT: [[LOAD_1_126529:%.*]] = extractelement <4 x b8> [[TMP1638]], i32 1
+; CHECK-NEXT: [[TMP1640:%.*]] = bitcast b8 [[LOAD_1_126529]] to i8
+; CHECK-NEXT: [[LOAD_2_126530:%.*]] = extractelement <4 x b8> [[TMP1638]], i32 2
+; CHECK-NEXT: [[TMP1641:%.*]] = bitcast b8 [[LOAD_2_126530]] to i8
+; CHECK-NEXT: [[LOAD_3_126531:%.*]] = extractelement <4 x b8> [[TMP1638]], i32 3
+; CHECK-NEXT: [[TMP1642:%.*]] = bitcast b8 [[LOAD_3_126531]] to i8
+; CHECK-NEXT: [[PTR2_0_126:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 504
+; CHECK-NEXT: [[TMP1643:%.*]] = bitcast i8 [[TMP1639]] to b8
+; CHECK-NEXT: [[TMP1644:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1643]], i32 0
+; CHECK-NEXT: [[TMP1645:%.*]] = bitcast i8 [[TMP1640]] to b8
+; CHECK-NEXT: [[TMP1646:%.*]] = insertelement <4 x b8> [[TMP1644]], b8 [[TMP1645]], i32 1
+; CHECK-NEXT: [[TMP1647:%.*]] = bitcast i8 [[TMP1641]] to b8
+; CHECK-NEXT: [[TMP1648:%.*]] = insertelement <4 x b8> [[TMP1646]], b8 [[TMP1647]], i32 2
+; CHECK-NEXT: [[TMP1649:%.*]] = bitcast i8 [[TMP1642]] to b8
+; CHECK-NEXT: [[TMP1650:%.*]] = insertelement <4 x b8> [[TMP1648]], b8 [[TMP1649]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1650]], ptr [[PTR2_0_126]], align 4
+; CHECK-NEXT: [[PTR_0_127:%.*]] = getelementptr i8, ptr [[GLOBAL_BASE_LOADS]], i64 508
+; CHECK-NEXT: [[TMP1651:%.*]] = load <4 x b8>, ptr [[PTR_0_127]], align 4
+; CHECK-NEXT: [[LOAD_0_127532:%.*]] = extractelement <4 x b8> [[TMP1651]], i32 0
+; CHECK-NEXT: [[TMP1652:%.*]] = bitcast b8 [[LOAD_0_127532]] to i8
+; CHECK-NEXT: [[LOAD_1_127533:%.*]] = extractelement <4 x b8> [[TMP1651]], i32 1
+; CHECK-NEXT: [[TMP1653:%.*]] = bitcast b8 [[LOAD_1_127533]] to i8
+; CHECK-NEXT: [[LOAD_2_127534:%.*]] = extractelement <4 x b8> [[TMP1651]], i32 2
+; CHECK-NEXT: [[TMP1654:%.*]] = bitcast b8 [[LOAD_2_127534]] to i8
+; CHECK-NEXT: [[LOAD_3_127535:%.*]] = extractelement <4 x b8> [[TMP1651]], i32 3
+; CHECK-NEXT: [[TMP1655:%.*]] = bitcast b8 [[LOAD_3_127535]] to i8
+; CHECK-NEXT: [[PTR2_0_127:%.*]] = getelementptr i8, ptr [[LOCAL_BASE_STORES]], i64 508
+; CHECK-NEXT: [[TMP1656:%.*]] = bitcast i8 [[TMP1652]] to b8
+; CHECK-NEXT: [[TMP1657:%.*]] = insertelement <4 x b8> poison, b8 [[TMP1656]], i32 0
+; CHECK-NEXT: [[TMP1658:%.*]] = bitcast i8 [[TMP1653]] to b8
+; CHECK-NEXT: [[TMP1659:%.*]] = insertelement <4 x b8> [[TMP1657]], b8 [[TMP1658]], i32 1
+; CHECK-NEXT: [[TMP1660:%.*]] = bitcast i8 [[TMP1654]] to b8
+; CHECK-NEXT: [[TMP1661:%.*]] = insertelement <4 x b8> [[TMP1659]], b8 [[TMP1660]], i32 2
+; CHECK-NEXT: [[TMP1662:%.*]] = bitcast i8 [[TMP1655]] to b8
+; CHECK-NEXT: [[TMP1663:%.*]] = insertelement <4 x b8> [[TMP1661]], b8 [[TMP1662]], i32 3
+; CHECK-NEXT: store <4 x b8> [[TMP1663]], ptr [[PTR2_0_127]], align 4
+; CHECK-NEXT: ret void
+;
entry:
; Create base pointer to a global variable with the inefficient pattern that Alias Analysis cannot easily traverse through.
%global_base_loads = getelementptr i8, ptr inttoptr (i32 ptrtoint (ptr @global_mem to i32) to ptr), i64 0
diff --git a/llvm/test/Transforms/LoadStoreVectorizer/int_sideeffect.ll b/llvm/test/Transforms/LoadStoreVectorizer/int_sideeffect.ll
index b20d3ca4f6029..2d1a2c215fda4 100644
--- a/llvm/test/Transforms/LoadStoreVectorizer/int_sideeffect.ll
+++ b/llvm/test/Transforms/LoadStoreVectorizer/int_sideeffect.ll
@@ -8,18 +8,26 @@ declare void @llvm.sideeffect()
define void @test_sideeffect(ptr %p) {
; CHECK-LABEL: @test_sideeffect(
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
-; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
-; CHECK-NEXT: [[L34:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr [[P:%.*]], align 16
+; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[L01]] to float
+; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[L12]] to float
+; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[L23]] to float
+; CHECK-NEXT: [[L34:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[L34]] to float
; CHECK-NEXT: call void @llvm.sideeffect()
; CHECK-NEXT: call void @llvm.sideeffect()
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> poison, float [[L01]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[L12]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[L23]], i32 2
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[L34]], i32 3
-; CHECK-NEXT: store <4 x float> [[TMP6]], ptr [[P]], align 16
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast float [[TMP2]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x b32> poison, b32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast float [[TMP3]] to b32
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x b32> [[TMP7]], b32 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast float [[TMP4]] to b32
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x b32> [[TMP9]], b32 [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast float [[TMP5]] to b32
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x b32> [[TMP11]], b32 [[TMP12]], i32 3
+; CHECK-NEXT: store <4 x b32> [[TMP13]], ptr [[P]], align 16
; CHECK-NEXT: ret void
;
%p1 = getelementptr float, ptr %p, i64 1
@@ -42,18 +50,26 @@ declare void @foo()
define void @test_inaccessiblememonly_nounwind_willreturn(ptr %p) {
; CHECK-LABEL: @test_inaccessiblememonly_nounwind_willreturn(
-; CHECK-NEXT: [[TMP2:%.*]] = load <4 x float>, ptr [[P:%.*]], align 16
-; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x float> [[TMP2]], i32 0
-; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x float> [[TMP2]], i32 1
-; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x float> [[TMP2]], i32 2
-; CHECK-NEXT: [[L34:%.*]] = extractelement <4 x float> [[TMP2]], i32 3
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x b32>, ptr [[P:%.*]], align 16
+; CHECK-NEXT: [[L01:%.*]] = extractelement <4 x b32> [[TMP1]], i32 0
+; CHECK-NEXT: [[TMP2:%.*]] = bitcast b32 [[L01]] to float
+; CHECK-NEXT: [[L12:%.*]] = extractelement <4 x b32> [[TMP1]], i32 1
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast b32 [[L12]] to float
+; CHECK-NEXT: [[L23:%.*]] = extractelement <4 x b32> [[TMP1]], i32 2
+; CHECK-NEXT: [[TMP4:%.*]] = bitcast b32 [[L23]] to float
+; CHECK-NEXT: [[L34:%.*]] = extractelement <4 x b32> [[TMP1]], i32 3
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast b32 [[L34]] to float
; CHECK-NEXT: call void @foo() #[[ATTR1:[0-9]+]]
; CHECK-NEXT: call void @foo() #[[ATTR1]]
-; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x float> poison, float [[L01]], i32 0
-; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x float> [[TMP3]], float [[L12]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> [[TMP4]], float [[L23]], i32 2
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> [[TMP5]], float [[L34]], i32 3
-; CHECK-NEXT: store <4 x float> [[TMP6]], ptr [[P]], align 16
+; CHECK-NEXT: [[TMP6:%.*]] = bitcast float [[TMP2]] to b32
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x b32> poison, b32 [[TMP6]], i32 0
+; CHECK-NEXT: [[TMP8:%.*]] = bitcast float [[TMP3]] to b32
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x b32> [[TMP7]], b32 [[TMP8]], i32 1
+; CHECK-NEXT: [[TMP10:%.*]] = bitcast float [[TMP4]] to b32
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x b32> [[TMP9]], b32 [[TMP10]], i32 2
+; CHECK-NEXT: [[TMP12:%.*]] = bitcast float [[TMP5]] to b32
+; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x b32> [[TMP11]], b32 [[TMP12]], i32 3
+; CHECK-NEXT: store <4 x b32> [[TMP13]], ptr [[P]], align 16
; CHECK-NEXT: ret void
;
%p1 = getelementptr float, ptr %p, i64 1
More information about the llvm-commits
mailing list