[llvm] 07aec21 - [AMDGPU] merge 16bit mov pairs in post-RA peephole (#208625)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Sep 25 09:56:17 PDT 2026
Author: Guo Chen
Date: 2026-09-25T12:56:10-04:00
New Revision: 07aec21c88f48907b5da1d42f4522e106d30a9f1
URL: https://github.com/llvm/llvm-project/commit/07aec21c88f48907b5da1d42f4522e106d30a9f1
DIFF: https://github.com/llvm/llvm-project/commit/07aec21c88f48907b5da1d42f4522e106d30a9f1.diff
LOG: [AMDGPU] merge 16bit mov pairs in post-RA peephole (#208625)
Address https://github.com/llvm/llvm-project/issues/207011. Add 5
patterns to merge 16bit mov pairs.
Since we hoist instruction which might cross the waitcnt boundary,
insert a new pass before waitcnt for this transformation
Co-Authored-By: Claude
Added:
llvm/lib/Target/AMDGPU/SIPostRA16BitMovFolding.cpp
llvm/test/CodeGen/AMDGPU/si-post-ra-merge-v-mov-b16.mir
Modified:
llvm/lib/Target/AMDGPU/AMDGPU.h
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
llvm/lib/Target/AMDGPU/CMakeLists.txt
llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
llvm/test/CodeGen/AMDGPU/add.v2i16.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index c87e9adeaa7d1..809540bd05b45 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -120,6 +120,7 @@ FunctionPass *createSIMemoryLegalizerPass();
FunctionPass *createSIInsertWaitcntsPass();
FunctionPass *createSIPreAllocateWWMRegsLegacyPass();
FunctionPass *createSIFormMemoryClausesLegacyPass();
+FunctionPass *createSIPostRA16BitMovFoldingPass();
FunctionPass *createSIPostRABundlerPass();
FunctionPass *createAMDGPUImageIntrinsicOptimizerPass(const TargetMachine *);
@@ -326,6 +327,9 @@ extern char &AMDGPUPreloadKernArgPrologLegacyID;
void initializeAMDGPUPreloadKernelArgumentsLegacyPass(PassRegistry &);
extern char &AMDGPUPreloadKernelArgumentsLegacyID;
+void initializeSIPostRA16BitMovFoldingLegacyPass(PassRegistry &);
+extern char &SIPostRA16BitMovFoldingLegacyID;
+
// Passes common to R600 and SI
FunctionPass *createAMDGPUPromoteAlloca();
void initializeAMDGPUPromoteAllocaPass(PassRegistry&);
@@ -535,6 +539,13 @@ class AMDGPUSetWavePriorityPass
MachineFunctionAnalysisManager &MFAM);
};
+class SIPostRA16BitMovFoldingPass
+ : public OptionalPassInfoMixin<SIPostRA16BitMovFoldingPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+
FunctionPass *createAMDGPUAnnotateUniformValuesLegacy();
ModulePass *createAMDGPUPrintfRuntimeBinding();
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 29e9046e4f8ca..372d5f5acab21 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -161,6 +161,7 @@ MACHINE_FUNCTION_PASS("si-optimize-exec-masking", SIOptimizeExecMaskingPass())
MACHINE_FUNCTION_PASS("si-optimize-exec-masking-pre-ra", SIOptimizeExecMaskingPreRAPass())
MACHINE_FUNCTION_PASS("si-peephole-sdwa", SIPeepholeSDWAPass())
MACHINE_FUNCTION_PASS("si-post-ra-bundler", SIPostRABundlerPass())
+MACHINE_FUNCTION_PASS("si-post-ra-16bit-mov-folding", SIPostRA16BitMovFoldingPass())
MACHINE_FUNCTION_PASS("si-pre-allocate-wwm-regs", SIPreAllocateWWMRegsPass())
MACHINE_FUNCTION_PASS("si-pre-emit-peephole", SIPreEmitPeepholePass())
MACHINE_FUNCTION_PASS("si-shrink-instructions", SIShrinkInstructionsPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 946d25400e17d..b47a000a1c980 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -768,6 +768,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeAMDGPUWaitSGPRHazardsLegacyPass(*PR);
initializeAMDGPUPreloadKernelArgumentsLegacyPass(*PR);
initializeAMDGPUUniformIntrinsicCombineLegacyPass(*PR);
+ initializeSIPostRA16BitMovFoldingLegacyPass(*PR);
}
static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -2033,6 +2034,8 @@ void GCNPassConfig::addPreEmitPass() {
if (isPassEnabled(EnableVOPD, CodeGenOptLevel::Less))
addPass(&GCNCreateVOPDID);
addPass(createSIMemoryLegalizerPass());
+ if (getOptLevel() > CodeGenOptLevel::None)
+ addPass(&SIPostRA16BitMovFoldingLegacyID);
addPass(createSIInsertWaitcntsPass());
addPass(createSIModeRegisterPass());
@@ -2778,6 +2781,8 @@ void AMDGPUCodeGenPassBuilder::addPreEmitPass(PassManagerWrapper &PMW) {
}
addMachineFunctionPass(SIMemoryLegalizerPass(), PMW);
+ if (TM.getOptLevel() > CodeGenOptLevel::None)
+ addMachineFunctionPass(SIPostRA16BitMovFoldingPass(), PMW);
addMachineFunctionPass(SIInsertWaitcntsPass(), PMW);
addMachineFunctionPass(SIModeRegisterPass(), PMW);
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index b7e679a69a80d..4a5f77d55afa5 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -191,6 +191,7 @@ add_llvm_target(AMDGPUCodeGen
SIRegisterInfo.cpp
SIShrinkInstructions.cpp
SIWholeQuadMode.cpp
+ SIPostRA16BitMovFolding.cpp
LINK_COMPONENTS
AMDGPUDesc
diff --git a/llvm/lib/Target/AMDGPU/SIPostRA16BitMovFolding.cpp b/llvm/lib/Target/AMDGPU/SIPostRA16BitMovFolding.cpp
new file mode 100644
index 0000000000000..31025ca1a7857
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/SIPostRA16BitMovFolding.cpp
@@ -0,0 +1,363 @@
+//===-- SIPostRA16BitMovFolding.cpp ------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This pass performs the post RA 16bit Mov folding
+///
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "MCTargetDesc/AMDGPUMCTargetDesc.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineLoopInfo.h"
+#include "llvm/CodeGen/MachinePostDominators.h"
+#include "llvm/CodeGen/TargetSchedule.h"
+#include "llvm/Support/BranchProbability.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "si-post-ra-16bit-mov-folding"
+
+namespace {
+
+class SIPostRA16BitMovFolding {
+private:
+ const SIInstrInfo *TII = nullptr;
+ const SIRegisterInfo *TRI = nullptr;
+
+ void getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
+ MCRegister &SrcReg16, bool &SrcIsVGPR,
+ MCRegister &SrcReg32, bool &SrcIsHi, bool &SrcIsImm,
+ int64_t &ImmVal) const;
+
+ bool mergeSingleMovB16Pair(MachineInstr &Lo, MachineInstr &Hi,
+ bool IsHiFirst) const;
+ bool mergeMovB16Pairs(MachineFunction &MF) const;
+
+public:
+ bool run(MachineFunction &MF);
+};
+
+class SIPostRA16BitMovFoldingLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ SIPostRA16BitMovFoldingLegacy() : MachineFunctionPass(ID) {}
+
+ StringRef getPassName() const override {
+ return "SI post-RA 16bit Mov Folding";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesAll();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+
+ bool runOnMachineFunction(MachineFunction &MF) override {
+ return SIPostRA16BitMovFolding().run(MF);
+ }
+};
+
+} // End anonymous namespace.
+
+INITIALIZE_PASS(SIPostRA16BitMovFoldingLegacy, DEBUG_TYPE,
+ "SI Post RA 16bit Mov Folding", false, false)
+
+char SIPostRA16BitMovFoldingLegacy::ID = 0;
+
+char &llvm::SIPostRA16BitMovFoldingLegacyID = SIPostRA16BitMovFoldingLegacy::ID;
+
+// Helper: extract the src operand and whether it is from the hi16 half.
+// Post-RA, both V_MOV_B16_t16_e32 and V_MOV_B16_t16_e64 use VGPR_16 dst
+// physical registers whose encoding already encodes hi/lo (IS_HI16 bit).
+void SIPostRA16BitMovFolding::getMovB16Info(
+ const MachineInstr &MI, const SIRegisterInfo *TRI, MCRegister &SrcReg16,
+ bool &SrcIsVGPR, MCRegister &SrcReg32, bool &SrcIsHi, bool &SrcIsImm,
+ int64_t &ImmVal) const {
+ SrcIsImm = false;
+ SrcIsHi = false;
+ SrcIsVGPR = false;
+ SrcReg16 = MCRegister();
+ SrcReg32 = MCRegister();
+
+ const MachineOperand *SrcOp = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
+
+ if (SrcOp->isImm()) {
+ SrcIsImm = true;
+ ImmVal = SrcOp->getImm();
+ return;
+ }
+
+ SrcReg16 = SrcOp->getReg().asMCReg();
+ SrcIsVGPR = AMDGPU::VGPR_16RegClass.contains(SrcReg16);
+ if (SrcIsVGPR) {
+ SrcIsHi = AMDGPU::isHi16Reg(SrcReg16, *TRI);
+ SrcReg32 = TRI->get32BitRegister(SrcReg16);
+ } else {
+ SrcIsHi = false;
+ SrcReg32 = SrcReg16;
+ }
+}
+
+// clang-format off
+// Try to merge a pair of v_mov_b16 instructions targeting the lo16 and hi16
+// halves of the same VGPR into a single 32-bit instruction.
+//
+// Caller guarantee the pair to be two v_mov_b16 and targets the same dst32
+//
+// Patterns:
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.l/s2 => v_and_b32 v0,0xffff,v2/s2
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.h => v_lshrrev_b32 v0,16,v2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.l/s2 => v_lshlrev_b32 v0,16,v2/s2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.h => v_and_b32 v0,0xffff0000,v2
+// v_mov_b16 v0.l, v.x/s v_mov_b16 v0.h, v.y/s => v_perm_b32_e64 v0, v.x/s, v.y/s, mask
+// clang-format on
+bool SIPostRA16BitMovFolding::mergeSingleMovB16Pair(MachineInstr &Lo,
+ MachineInstr &Hi,
+ bool IsHiFirst) const {
+ // Lo and Hi share the same Dst32
+ MCRegister LoDst = Lo.getOperand(0).getReg().asMCReg();
+ MCRegister HiDst = Hi.getOperand(0).getReg().asMCReg();
+ MCRegister Dst32 = TRI->get32BitRegister(LoDst);
+
+ // Extract source info for Lo and Hi.
+ MCRegister LoSrc16, LoSrc32, HiSrc16, HiSrc32;
+ bool LoSrcIsHi, HiSrcIsHi, LoSrcIsImm, HiSrcIsImm, LoSrcIsVGPR, HiSrcIsVGPR;
+ int64_t LoImm = 0, HiImm = 0;
+
+ getMovB16Info(Lo, TRI, LoSrc16, LoSrcIsVGPR, LoSrc32, LoSrcIsHi, LoSrcIsImm,
+ LoImm);
+ getMovB16Info(Hi, TRI, HiSrc16, HiSrcIsVGPR, HiSrc32, HiSrcIsHi, HiSrcIsImm,
+ HiImm);
+
+ MachineInstr &FirstMI = IsHiFirst ? Hi : Lo;
+ MachineInstr &SecondMI = IsHiFirst ? Lo : Hi;
+
+ // Data Conflict counter
+ MachineBasicBlock::iterator UpperBound = SecondMI.getIterator();
+ MachineBasicBlock::iterator LowerBound = FirstMI.getIterator();
+ unsigned LoopCnt = 0, UpperBoundCnt = UINT_MAX, LowerBoundCnt = 0;
+
+ MachineBasicBlock &MBB = *Lo.getParent();
+
+ // Check that between Lo and Hi, there are no instructions that:
+ // - modify Dst32
+ // - modify LoSrc16 or HiSrc16 depending on order (data dependency)
+ // We scan from the instruction after the first mov up to (but not including)
+ // the second mov.
+ MCRegister FirstSrc16 = IsHiFirst ? HiSrc16 : LoSrc16;
+ MCRegister FirstDst16 = IsHiFirst ? HiDst : LoDst;
+ MCRegister SecondSrc16 = IsHiFirst ? LoSrc16 : HiSrc16;
+ MCRegister SecondDst16 = IsHiFirst ? LoDst : HiDst;
+ for (MachineInstr &Scan :
+ drop_begin(make_range(FirstMI.getIterator(), SecondMI.getIterator()))) {
+ if (Scan.isDebugInstr())
+ continue;
+ if (Scan.modifiesRegister(Dst32, TRI))
+ return false;
+ assert(!Scan.modifiesRegister(AMDGPU::EXEC, TRI) &&
+ "Expect no write on EXEC!");
+ LoopCnt++;
+ if (LoopCnt < UpperBoundCnt &&
+ ((FirstSrc16 && Scan.modifiesRegister(FirstSrc16, TRI)) ||
+ Scan.readsRegister(FirstDst16, TRI))) {
+ UpperBound = Scan.getIterator();
+ UpperBoundCnt = LoopCnt;
+ }
+ if (LoopCnt > LowerBoundCnt &&
+ ((SecondSrc16 && Scan.modifiesRegister(SecondSrc16, TRI)) ||
+ Scan.readsRegister(SecondDst16, TRI))) {
+ LowerBound = Scan.getIterator();
+ LowerBoundCnt = LoopCnt;
+ }
+ }
+
+ // No spot maintains data dependency
+ if (LowerBoundCnt >= UpperBoundCnt)
+ return false;
+
+ // Insert MI before selected. Any spots between (LowerBound, UpperBound] would
+ // work
+ MachineInstr &Selected = *UpperBound;
+ const DebugLoc &DL = Selected.getDebugLoc();
+
+ // Now match patterns and emit the replacement instruction.
+ // Insert on Selected MI location, then remove both mov.
+
+ // Pattern: v_mov_b16 v0.l, v2.x/s2 + v_mov_b16 v0.h, v3.y/s3
+ // => v_perm_b32_e64 v0,v3.y/s3,v2.x/s2, mask
+ if (!HiSrcIsImm && !LoSrcIsImm) {
+ // Violate constant bus restriction
+ if (!LoSrcIsVGPR && !HiSrcIsVGPR && HiSrc32 != LoSrc32)
+ return false;
+ unsigned MaskHiSrc = HiSrcIsHi ? 0x0706 : 0x0504;
+ unsigned MaskLoSrc = LoSrcIsHi ? 0x0302 : 0x0100;
+ BuildMI(MBB, Selected, DL, TII->get(AMDGPU::V_PERM_B32_e64), Dst32)
+ .addReg(HiSrc32)
+ .addReg(LoSrc32)
+ .addImm((MaskHiSrc << 16) | MaskLoSrc);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ bool Usevop2 =
+ AMDGPU::VGPR_32_Lo128RegClass.contains(Dst32) &&
+ (LoSrcIsImm ||
+ (LoSrcIsVGPR && AMDGPU::VGPR_32_Lo128RegClass.contains(LoSrc32))) &&
+ (HiSrcIsImm ||
+ (HiSrcIsVGPR && AMDGPU::VGPR_32_Lo128RegClass.contains(HiSrc32)));
+
+ // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l/s2
+ // => v_and_b32 v0, 0x0000ffff, v2/s2
+ if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && !LoSrcIsHi) {
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_AND_B32_e32 : AMDGPU::V_AND_B32_e64),
+ Dst32)
+ .addImm(0x0000ffff)
+ .addReg(LoSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.h
+ // => v_lshrrev_b32 v0, 16, v2
+ if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && LoSrcIsHi) {
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_LSHRREV_B32_e32
+ : AMDGPU::V_LSHRREV_B32_e64),
+ Dst32)
+ .addImm(16)
+ .addReg(LoSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l/s2
+ // => v_lshlrev_b32 v0, 16, v2/s2
+ if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && !HiSrcIsHi) {
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_LSHLREV_B32_e32
+ : AMDGPU::V_LSHLREV_B32_e64),
+ Dst32)
+ .addImm(16)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
+ // => v_and_b32 v0, 0xffff0000, v2
+ if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && HiSrcIsHi) {
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_AND_B32_e32 : AMDGPU::V_AND_B32_e64),
+ Dst32)
+ .addImm(0xffff0000)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ return false;
+}
+
+// Merge pairs of v_mov_b16 targeting the lo16 and hi16 halves of the same
+// VGPR into a single 32-bit instruction (true16 mode only).
+bool SIPostRA16BitMovFolding::mergeMovB16Pairs(MachineFunction &MF) const {
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ // Map from 32-bit VGPR to the pending v_mov_b16 and its age.
+ // Tracks how many non-mov-b16 instructions have passed since the
+ // 16-bit write using a fixed size circular buffer
+ struct Pending {
+ MCRegister Dst32;
+ MachineInstr *MI;
+ unsigned IsHi;
+ };
+ // Search window size
+ const unsigned ScanLimit = 16;
+ std::array<Pending, ScanLimit> CirBuf = {};
+ SmallDenseMap<MCRegister, unsigned> PendingWrites;
+ unsigned Head = 0;
+
+ for (MachineInstr &MI : make_early_inc_range(MBB)) {
+ if (MI.isDebugInstr())
+ continue;
+
+ unsigned Opc = MI.getOpcode();
+ bool IsMovB16 = (Opc == AMDGPU::V_MOV_B16_t16_e32 ||
+ Opc == AMDGPU::V_MOV_B16_t16_e64);
+
+ if (++Head == ScanLimit)
+ Head = 0;
+
+ // Expire the last one
+ PendingWrites.erase(CirBuf[Head].Dst32);
+
+ if (!IsMovB16) {
+ CirBuf[Head] = {MCRegister(), nullptr, false};
+ continue;
+ }
+
+ LLVM_DEBUG(dbgs() << "Checking MI:" << MI << "\n");
+ MCRegister DstReg = MI.getOperand(0).getReg().asMCReg();
+ bool DstIsHi = AMDGPU::isHi16Reg(DstReg, *TRI);
+ MCRegister Dst32 = TRI->get32BitRegister(DstReg);
+
+ // Insert new one
+ CirBuf[Head] = {Dst32, &MI, DstIsHi};
+
+ auto [It, Inserted] = PendingWrites.insert({Dst32, Head});
+ if (!Inserted) {
+ if (CirBuf[It->second].IsHi == DstIsHi) {
+ It->second = Head;
+ continue;
+ }
+
+ // Look for a matching pending write.
+ MachineInstr &LoMI = !DstIsHi ? MI : *CirBuf[It->second].MI;
+ MachineInstr &HiMI = DstIsHi ? MI : *CirBuf[It->second].MI;
+ bool IsHiFirst = CirBuf[It->second].IsHi;
+ if (mergeSingleMovB16Pair(LoMI, HiMI, IsHiFirst)) {
+ Changed = true;
+ PendingWrites.erase(It);
+ } else {
+ It->second = Head;
+ }
+ }
+ }
+ }
+
+ return Changed;
+}
+
+PreservedAnalyses
+llvm::SIPostRA16BitMovFoldingPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+ SIPostRA16BitMovFolding().run(MF);
+ return PreservedAnalyses::all();
+}
+
+bool SIPostRA16BitMovFolding::run(MachineFunction &MF) {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
+ TII = ST.getInstrInfo();
+ bool Changed = false;
+
+ // Try merge B16 Pair in true16 mode
+ if (ST.useRealTrue16Insts())
+ Changed |= mergeMovB16Pairs(MF);
+
+ return Changed;
+}
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
index dbb753c0b3072..77c589ca3ceeb 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/llvm.amdgcn.intersect_ray.ll
@@ -69,10 +69,9 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16(i32 %node_ptr, float %
; GFX11-TRUE16-LABEL: image_bvh_intersect_ray_a16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v9, v5 :: v_dual_mov_b32 v10, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v6.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v9.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v9, v7, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v7.h
; GFX11-TRUE16-NEXT: image_bvh_intersect_ray v[0:3], [v0, v1, v[2:4], v[8:10]], s[0:3] a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -145,10 +144,9 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16(i64 %node_ptr, float
; GFX11-TRUE16-LABEL: image_bvh64_intersect_ray_a16:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v10, v6 :: v_dual_mov_b32 v11, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v7.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v10.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v10, v8, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.h
; GFX11-TRUE16-NEXT: image_bvh64_intersect_ray v[0:3], [v[0:1], v2, v[3:5], v[9:11]], s[0:3] a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -367,8 +365,8 @@ define amdgpu_ps <4 x float> @image_bvh_intersect_ray_a16_vgpr_descr(i32 %node_p
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v16, v0 :: v_dual_mov_b32 v17, v1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v13, v2 :: v_dual_mov_b32 v14, v3
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v15, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v7.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v19.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v19, v7, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v7.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v6.l
; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
@@ -638,9 +636,8 @@ define amdgpu_ps <4 x float> @image_bvh64_intersect_ray_a16_vgpr_descr(i64 %node
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v5, v6 :: v_dual_mov_b32 v6, v9
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v17, v0 :: v_dual_mov_b32 v18, v1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v19, v2 :: v_dual_mov_b32 v14, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v8.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v5, v8, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v8.h
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v7.l
; GFX11-TRUE16-NEXT: s_mov_b32 s1, exec_lo
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
index fc1be777b09a3..01c8e4cb6dab7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/merge-values-s16-true16.ll
@@ -36,9 +36,8 @@ define amdgpu_kernel void @store_i136_divergent(ptr %p) {
; GFX1150-NEXT: v_mov_b16_e32 v0.h, 0
; GFX1150-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX1150-NEXT: v_lshlrev_b16 v4.l, 8, v1.l
-; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1150-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX1150-NEXT: v_mov_b16_e32 v1.h, v0.h
+; GFX1150-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1150-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX1150-NEXT: v_or_b16 v0.l, v0.l, v4.l
; GFX1150-NEXT: s_waitcnt lgkmcnt(0)
; GFX1150-NEXT: v_dual_mov_b32 v5, s1 :: v_dual_mov_b32 v4, s0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
index aee2cd8428427..56dcedb7977a7 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/saddsat.ll
@@ -768,16 +768,12 @@ define i32 @v_saddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v2, v0, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v4, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v0, 0x5040302
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v2, 8, v6 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v3, 8, v7 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v5, v1, 0x5040302
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_i16 v2, v2, v3 clamp
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
index e23337d7ae9ec..e402f24efa074 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/ssubsat.ll
@@ -768,16 +768,12 @@ define i32 @v_ssubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v2, v0, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v4, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v0, 0x5040302
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v2, 8, v6 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v3, 8, v7 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v5, v1, 0x5040302
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_sub_i16 v2, v2, v3 clamp
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
index 163698d594dc5..8b77812f28f9c 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/uaddsat.ll
@@ -665,16 +665,12 @@ define i32 @v_uaddsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v2, v0, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v4, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v0, 0x5040302
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v2, 8, v6 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v3, 8, v7 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v5, v1, 0x5040302
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_add_u16 v2, v2, v3 clamp
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
index 2d3abaf51da69..bee44a60b9be8 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/usubsat.ll
@@ -653,16 +653,12 @@ define i32 @v_usubsat_v4i8(i32 %lhs.arg, i32 %rhs.arg) {
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v0
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 24, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v2, v0, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v4, v1, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v0, 0x5040302
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v2, 8, v6 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v3, 8, v7 op_sel_hi:[0,1]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v5.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v5, v1, 0x5040302
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v0, 8, v0 op_sel_hi:[0,1]
; GFX11-TRUE16-NEXT: v_pk_sub_u16 v2, v2, v3 clamp
; GFX11-TRUE16-NEXT: v_pk_lshlrev_b16 v1, 8, v1 op_sel_hi:[0,1]
diff --git a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
index 45c558842188c..bc2380d2cea88 100644
--- a/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/add.v2i16.ll
@@ -767,9 +767,8 @@ define amdgpu_kernel void @v_test_add_v2i16_zext_to_v2i64(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: global_load_b32 v0, v0, s[4:5] glc dlc
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_pk_add_u16 v0, v2, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040302
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
index 37436f1d94c96..88ef88da73bce 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.1024bit.ll
@@ -164426,9 +164426,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v50.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v51.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v51, v50, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v5, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v6, v8
@@ -164461,8 +164460,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v10, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v54.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v55.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v55, v54, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v7, v8, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v6, v6
@@ -164498,12 +164497,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_bfe_u32 v11, v12, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v10
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v65, 16, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v67.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v10, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_or_b32_e32 v9, 0x400000, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v65.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v67, v65, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v68, 16, v7
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v8, v10
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v8, v11, v12
@@ -164609,12 +164608,12 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v20, 0x400000, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v81, 16, v12
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.h, v97.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v13, v13, v14, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v14, v16, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v15, v15
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v12.l, v81.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v12, v97, v81, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v86, 16, v13
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v13, v14, v16
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v14, v17, v18
@@ -164702,8 +164701,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v23
; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v22, 16, 1
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v112.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v102.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v113.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v113, v102, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
@@ -164722,8 +164721,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v22
; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v23, 16, 1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s2, 16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v87.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v101.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v101, v87, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
@@ -164742,8 +164741,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v23
; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v22, 16, 1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s17, 16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v64.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v66.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v66, v64, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
@@ -164762,8 +164761,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v22
; GFX11-TRUE16-NEXT: v_bfe_u32 v22, v23, 16, 1
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s16, 16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v116.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v36.h, v129.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v36, v129, v116, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
@@ -164782,8 +164781,8 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v23
; GFX11-TRUE16-NEXT: v_bfe_u32 v23, v22, 16, 1
; GFX11-TRUE16-NEXT: s_and_b32 s1, s19, 0xffff0000
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v118.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v35.h, v132.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v35, v132, v118, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v19, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
@@ -164802,19 +164801,19 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v22
; GFX11-TRUE16-NEXT: v_or_b32_e32 v22, 0x400000, v24
; GFX11-TRUE16-NEXT: s_lshl_b32 s1, s26, 16
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.l, v114.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v30.h, v117.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v30, v117, v114, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v17, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v17, v18, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v19, v19
; GFX11-TRUE16-NEXT: v_bfe_u32 v19, v24, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v115.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v130, 16, v16
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, v17, v18
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v15, v15, v23, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v17, 0x400000, v18
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v18, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v29.h, v128.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v29, v128, v115, 0x5040100
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v16, 0x7fff, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v144, 16, v15
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v15, v19, v24
@@ -164949,8 +164948,7 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v16, v16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v100, 16, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v96, 16, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v43.l, v134.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v43.h, v149.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v43, v149, v134, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v18, v19, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, v22, v15
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v160, 16, v17
@@ -164981,50 +164979,36 @@ define inreg <128 x i8> @bitcast_v64bf16_to_v128i8_scalar(<64 x bfloat> inreg %a
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, v18, v23
; GFX11-TRUE16-NEXT: v_or_b32_e32 v26, 0x400000, v24
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v176, 16, v17
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v44.l, v150.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v16, v16, v19, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v24, v24
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v18, 0x7fff, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v44.h, v163.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v42.l, v146.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v44, v163, v150, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v162, 16, v16
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v19, v25, v26, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v23, v23
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v42.h, v151.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.l, v98.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v13.h, v103.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v42, v151, v146, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v13, v103, v98, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v165, 16, v19
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v18, v18, v27, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v41.l, v131.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v41.h, v145.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v40.l, v135.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v179.l, v162.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v41, v145, v131, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v177, 16, v18
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v179.h, v176.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.l, v165.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v40.h, v148.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v82.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v178.h, v177.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.h, v83.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v86.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v100.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v119.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v49.h, v133.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v48.l, v130.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v48.h, v144.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v183.l, v160.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v183.h, v164.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v182.l, v166.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v182.h, v167.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v179, v176, v162, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v40, v148, v135, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_perm_b32 v178, v177, v165, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v7, v83, v82, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v100, v86, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v49, v133, v119, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v48, v144, v130, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v183, v164, v160, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v182, v167, v166, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v53.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v68.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v69.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v80.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v84.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v85.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v96.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v45.l, v147.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e64 v45.h, v161.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v69, v68, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v84, v80, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v96, v85, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v45, v161, v147, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v73, 8, v44
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v74, 8, v43
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[31:32], 24, v[42:43]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
index ba204b2b80688..4d9eb297b8313 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.128bit.ll
@@ -26077,7 +26077,7 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v9, 16, 1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v8, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v17.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v1, v4, 16, 1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, v10, v9
@@ -26091,12 +26091,10 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v6.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v6, v17, 0x5040100
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v1, 0x7fff, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v18.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.h, v14.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_perm_b32 v18, v2, v0, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v19
@@ -26105,19 +26103,17 @@ define inreg <16 x i8> @bitcast_v8bf16_to_v16i8_scalar(<8 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v18
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v21.l, v16.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v21, v14, v16, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v4
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[18:19]
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 24, v21
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.l, v8.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v13, 8, v21
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v5, 8, v19
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v20.h, v10.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v20, v10, v8, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[20:21]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v20
; GFX11-TRUE16-NEXT: s_branch .LBB109_6
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
index 9a60f77fc3609..d9e5e138fb87c 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.256bit.ll
@@ -33953,21 +33953,17 @@ define <16 x i16> @bitcast_v32i8_to_v16i16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v27, v21, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v25, v15, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v13, 0x5040100
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v11, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
@@ -39461,21 +39457,17 @@ define <16 x half> @bitcast_v32i8_to_v16f16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v27, v21, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v25, v15, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v13, 0x5040100
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v11, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
@@ -42815,8 +42807,7 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v8, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v12, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v36, v2, v0, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, v5, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v34, 16, v4
@@ -42893,30 +42884,29 @@ define inreg <32 x i8> @bitcast_v16bf16_to_v32i8_scalar(<16 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v11
; GFX11-TRUE16-NEXT: v_or_b32_e32 v15, 0x400000, v12
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v30, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.l, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v12, v12
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, 0x7fff, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.l, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v49.h, v22.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v49, v22, v33, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v32, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v15, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.h, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.h, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v51.l, v32.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_perm_b32 v51, v30, v32, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v24, 16, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v16, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v16, 16, v1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v38.h, v10.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v38, v10, v8, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[36:37]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.l, v24.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v48.l, v16.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v48, v18, v16, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[38:39]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v31, 24, v51
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v29, 8, v51
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v50.h, v26.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v50, v26, v24, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[19:20], 24, v[48:49]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 24, v49
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v21, 8, v49
@@ -43870,21 +43860,17 @@ define <16 x bfloat> @bitcast_v32i8_to_v16bf16(<32 x i8> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: scratch_load_b32 v37, off, s32 offset:4
; GFX11-TRUE16-NEXT: scratch_load_d16_hi_b16 v31, off, s32
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.h, v27.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.h, v25.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v23.h, v23.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.h, v19.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.h, v17.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v35.l, v29.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v36.l, v28.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v27.l, v21.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v27, v27, v21, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v34.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v25.l, v15.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v25, v25, v15, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v32.l, v14.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v19.l, v13.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v19, v19, v13, 0x5040100
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v31.l, v12.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v17.l, v11.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v17, v17, v11, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v28.l, v10.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v29.l, v9.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v33.l, v8.l
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
index 48259e19a00c0..dc584970e1e0b 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.32bit.ll
@@ -10261,10 +10261,8 @@ define inreg <4 x i8> @bitcast_v2bf16_to_v4i8_scalar(<2 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v0, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
index ab21a7fa9cfe7..31400ccf4c903 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.48bit.ll
@@ -927,15 +927,15 @@ define <3 x i16> @bitcast_v3bf16_to_v3i16(<3 x bfloat> %a, i32 %b) #0 {
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v1, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v2, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v2, v3, v7 :: v_dual_mov_b32 v3, 0x7fc0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v4, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v1, 0x5040302
; GFX11-TRUE16-NEXT: .LBB4_2: ; %end
; GFX11-TRUE16-NEXT: s_or_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1184,15 +1184,13 @@ define inreg <3 x i16> @bitcast_v3bf16_to_v3i16_scalar(<3 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v3, 0x7fff, v3
; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v3, v3, v7 :: v_dual_add_nc_u32 v4, 0x7fff, v4
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v4, v6, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0x7fc0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v5, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v3.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v4.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v4, v1, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
; GFX11-TRUE16-NEXT: .LBB5_4:
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
index 7a73205e77008..f79e89b17eee4 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.512bit.ll
@@ -90323,9 +90323,8 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v5, v1, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v7, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v16.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v17.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v17, v16, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v4, v5, v7
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v5, v6, v8
@@ -90395,12 +90394,12 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_add_f32_e64 v11, 0x40c00000, s0
; GFX11-TRUE16-NEXT: s_lshl_b32 s0, s18, 16
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v6
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v25.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v7, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v10, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v9, v9
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v13, 16, 1
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v23.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v25, v23, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v26, 16, v7
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, v8, v10
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v5, v12, vcc_lo
@@ -90539,8 +90538,7 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v8, v8
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v27, 16, v5
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v69.h, v39.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.l, v34.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v81.h, v37.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v81, v37, v34, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v10, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, v12, v7
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v49, 16, v9
@@ -90573,39 +90571,30 @@ define inreg <64 x i8> @bitcast_v32bf16_to_v64i8_scalar(<32 x bfloat> inreg %a,
; GFX11-TRUE16-NEXT: v_or_b32_e32 v51, 0x400000, v14
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v10, 0x7fff, v10
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v53, 16, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.l, v55.l
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v8, v8, v11, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v14, v14
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v68.h, v50.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.h, v53.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.l, v48.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v80.h, v38.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v68, v50, v55, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v80, v38, v48, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v11, v15, v51, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v13, v13
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v51, 16, v8
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.l, v30.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v84.h, v33.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.l, v35.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v84, v33, v30, 0x5040100
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v10, v10, v52, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v52, 16, v11
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v67.l, v51.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v83.h, v36.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.l, v28.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_perm_b32 v67, v53, v51, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v83, v36, v35, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v54, 16, v10
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.l, v52.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v86.h, v29.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.l, v31.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v85.h, v32.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v66.h, v54.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v26.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v27.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v86, v29, v28, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v85, v32, v31, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_perm_b32 v66, v54, v52, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v27, v26, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v20.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v22.l
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[7:8], 24, v[66:67]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v24.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v24, v22, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[8:9], 24, v[68:69]
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v18.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v19.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v19, v18, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[9:10], 24, v[80:81]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[10:11], 24, v[83:84]
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[85:86]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
index 69727ac65ee4b..dc7e21b2b5a2d 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.64bit.ll
@@ -18088,15 +18088,12 @@ define inreg <8 x i8> @bitcast_v4bf16_to_v8i8_scalar(<4 x bfloat> inreg %a, i32
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v6, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.l, v8.l
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_perm_b32 v9, v2, v0, 0x5040100
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v9
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v10.h, v6.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v10, v6, v8, 0x5040100
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[9:10]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v10
diff --git a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
index ec71f29615daf..413ff619dddfb 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgcn.bitcast.96bit.ll
@@ -8754,8 +8754,8 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v7, 0x7fff, v7
; GFX11-TRUE16-NEXT: v_bfe_u32 v8, v3, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v10, 0x400000, v2
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.l, v13.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v15.h, v6.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v15, v6, v13, 0x5040100
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v7, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_or_b32_e32 v7, 0x400000, v5
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v5, v5
@@ -8772,24 +8772,21 @@ define inreg <12 x i8> @bitcast_v6bf16_to_v12i8_scalar(<6 x bfloat> inreg %a, i3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v3, v3
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 24, v15
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v3, v8, v9, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v2, v2
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.l, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 16, v3
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v1, v10, vcc_lo
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v14.h, v2.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.l, v8.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_perm_b32 v14, v2, v0, 0x5040100
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v10, 16, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[3:4], 24, v[14:15]
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v14
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, v13
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v11.h, v10.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v11, v10, v8, 0x5040100
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 8, v11
; GFX11-TRUE16-NEXT: v_lshrrev_b64 v[11:12], 24, v[11:12]
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index 5fc001c848575..0ee28c50db7ed 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -5463,10 +5463,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi(ptr inreg %sbase,
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v[0:1]
-; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi:
@@ -5544,10 +5542,8 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_zero_hi_immneg128(ptr inr
; GFX1250-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-SDAG-TRUE16-NEXT: v_add_nc_u64_e32 v[0:1], s[2:3], v[0:1]
; GFX1250-SDAG-TRUE16-NEXT: flat_load_u16 v1, v[0:1] offset:-128
-; GFX1250-SDAG-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.l, 0
; GFX1250-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-SDAG-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v1
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
; GFX1250-NOECC-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_zero_hi_immneg128:
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 97f6b7537658f..e346eb0a3b072 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -1,987 +1,486 @@
-; RUN: llc -enable-new-pm -mtriple=amdgpu7.00--amdhsa -O0 -global-isel=0 -print-pipeline-passes=tree < %s 2>&1 \
-; RUN: | FileCheck -check-prefix=GCN-O0-SDAG %s
+; RUN: llc -enable-new-pm -mtriple=amdgpu7.00--amdhsa -O0 -print-pipeline-passes=tree < %s 2>&1 \
+; RUN: | FileCheck -check-prefix=GCN-O0 %s
-; RUN: llc -enable-new-pm -mtriple=amdgpu7.00--amdhsa -O0 -global-isel=1 -print-pipeline-passes=tree < %s 2>&1 \
-; RUN: | FileCheck -check-prefix=GCN-O0-GISEL %s
+; RUN: llc -enable-new-pm -mtriple=amdgpu7.00--amdhsa -print-pipeline-passes=tree < %s 2>&1 \
+; RUN: | FileCheck -check-prefix=GCN-O2 %s
-; RUN: llc -enable-new-pm -mtriple=amdgpu7.00--amdhsa -global-isel=0 -print-pipeline-passes=tree < %s 2>&1 \
-; RUN: | FileCheck -check-prefix=GCN-O2-SDAG %s
+; RUN: llc -O3 -enable-new-pm -mtriple=amdgpu7.00--amdhsa -print-pipeline-passes=tree < %s 2>&1 \
+; RUN: | FileCheck -check-prefix=GCN-O3 %s
-; RUN: llc -enable-new-pm -mtriple=amdgpu7.00--amdhsa -global-isel=1 -print-pipeline-passes=tree < %s 2>&1 \
-; RUN: | FileCheck -check-prefix=GCN-O2-GISEL %s
+; GCN-O0: require<MachineModuleAnalysis>
+; GCN-O0-NEXT: require<profile-summary>
+; GCN-O0-NEXT: require<collector-metadata>
+; GCN-O0-NEXT: require<runtime-libcall-info>
+; GCN-O0-NEXT: require<libcall-lowering-info>
+; GCN-O0-NEXT: pre-isel-intrinsic-lowering
+; GCN-O0-NEXT: function
+; GCN-O0-NEXT: expand-ir-insts<O0>
+; GCN-O0-NEXT: amdgpu-remove-incompatible-functions
+; GCN-O0-NEXT: amdgpu-printf-runtime-binding
+; GCN-O0-NEXT: amdgpu-lower-ctor-dtor
+; GCN-O0-NEXT: function
+; GCN-O0-NEXT: amdgpu-uniform-intrinsic-combine
+; GCN-O0-NEXT: expand-variadics
+; GCN-O0-NEXT: amdgpu-always-inline
+; GCN-O0-NEXT: always-inline
+; GCN-O0-NEXT: amdgpu-export-kernel-runtime-handles
+; GCN-O0-NEXT: amdgpu-lower-exec-sync
+; GCN-O0-NEXT: amdgpu-sw-lower-lds
+; GCN-O0-NEXT: amdgpu-lower-module-lds
+; GCN-O0-NEXT: function
+; GCN-O0-NEXT: atomic-expand
+; GCN-O0-NEXT: verify
+; GCN-O0-NEXT: unreachableblockelim
+; GCN-O0-NEXT: ee-instrument<post-inline>
+; GCN-O0-NEXT: scalarize-masked-mem-intrin
+; GCN-O0-NEXT: expand-reductions
+; GCN-O0-NEXT: amdgpu-lower-kernel-arguments
+; GCN-O0-NEXT: amdgpu-lower-buffer-fat-pointers
+; GCN-O0-NEXT: amdgpu-lower-intrinsics
+; GCN-O0-NEXT: cgscc
+; GCN-O0-NEXT: function
+; GCN-O0-NEXT: lower-switch
+; GCN-O0-NEXT: lower-invoke
+; GCN-O0-NEXT: unreachableblockelim
+; GCN-O0-NEXT: amdgpu-unify-divergent-exit-nodes
+; GCN-O0-NEXT: fix-irreducible
+; GCN-O0-NEXT: unify-loop-exits
+; GCN-O0-NEXT: StructurizeCFGPass
+; GCN-O0-NEXT: amdgpu-annotate-uniform
+; GCN-O0-NEXT: si-annotate-control-flow
+; GCN-O0-NEXT: amdgpu-rewrite-undef-for-phi
+; GCN-O0-NEXT: lcssa
+; GCN-O0-NEXT: inline-asm-prepare
+; GCN-O0-NEXT: safe-stack
+; GCN-O0-NEXT: stack-protector
+; GCN-O0-NEXT: verify
+; GCN-O0-NEXT: amdgpu-asm-printer-begin
+; GCN-O0-NEXT: cgscc
+; GCN-O0-NEXT: function
+; GCN-O0-NEXT: machine-function
+; GCN-O0-NEXT: amdgpu-isel
+; GCN-O0-NEXT: si-fix-sgpr-copies
+; GCN-O0-NEXT: si-i1-copies
+; GCN-O0-NEXT: finalize-isel
+; GCN-O0-NEXT: localstackalloc
+; GCN-O0-NEXT: require<reg-usage>
+; GCN-O0-NEXT: cgscc
+; GCN-O0-NEXT: function
+; GCN-O0-NEXT: machine-function
+; GCN-O0-NEXT: reg-usage-propagation
+; GCN-O0-NEXT: phi-node-elimination
+; GCN-O0-NEXT: si-lower-control-flow
+; GCN-O0-NEXT: two-address-instruction
+; GCN-O0-NEXT: si-wqm
+; GCN-O0-NEXT: amdgpu-pre-ra-long-branch-reg
+; GCN-O0-NEXT: regallocfast<filter=sgpr;no-clear-vregs>
+; GCN-O0-NEXT: si-lower-sgpr-spills
+; GCN-O0-NEXT: si-pre-allocate-wwm-regs
+; GCN-O0-NEXT: regallocfast<filter=wwm;no-clear-vregs>
+; GCN-O0-NEXT: si-lower-wwm-copies
+; GCN-O0-NEXT: amdgpu-reserve-wwm-regs
+; GCN-O0-NEXT: regallocfast<filter=vgpr>
+; GCN-O0-NEXT: si-fix-vgpr-copies
+; GCN-O0-NEXT: remove-redundant-debug-values
+; GCN-O0-NEXT: fixup-statepoint-caller-saved
+; GCN-O0-NEXT: prolog-epilog
+; GCN-O0-NEXT: post-ra-pseudos
+; GCN-O0-NEXT: si-post-ra-bundler
+; GCN-O0-NEXT: fentry-insert
+; GCN-O0-NEXT: xray-instrumentation
+; GCN-O0-NEXT: si-memory-legalizer
+; GCN-O0-NEXT: si-insert-waitcnts
+; GCN-O0-NEXT: si-mode-register
+; GCN-O0-NEXT: si-late-branch-lowering
+; GCN-O0-NEXT: post-RA-hazard-rec
+; GCN-O0-NEXT: amdgpu-wait-sgpr-hazards
+; GCN-O0-NEXT: amdgpu-lower-vgpr-encoding
+; GCN-O0-NEXT: branch-relaxation
+; GCN-O0-NEXT: reg-usage-collector
+; GCN-O0-NEXT: remove-loads-into-fake-uses
+; GCN-O0-NEXT: live-debug-values
+; GCN-O0-NEXT: machine-sanmd
+; GCN-O0-NEXT: amdgpu-preload-kern-arg-prolog
+; GCN-O0-NEXT: stack-frame-layout
+; GCN-O0-NEXT: amdgpu-asm-printer
+; GCN-O0-NEXT: free-machine-function
+; GCN-O0-NEXT: amdgpu-asm-printer-end
-; RUN: llc -O3 -enable-new-pm -mtriple=amdgpu7.00--amdhsa -global-isel=0 -print-pipeline-passes=tree < %s 2>&1 \
-; RUN: | FileCheck -check-prefix=GCN-O3-SDAG %s
+; GCN-O2: require<MachineModuleAnalysis>
+; GCN-O2-NEXT: require<profile-summary>
+; GCN-O2-NEXT: require<collector-metadata>
+; GCN-O2-NEXT: require<runtime-libcall-info>
+; GCN-O2-NEXT: require<libcall-lowering-info>
+; GCN-O2-NEXT: function
+; GCN-O2-NEXT: objc-arc-contract
+; GCN-O2-NEXT: pre-isel-intrinsic-lowering
+; GCN-O2-NEXT: function
+; GCN-O2-NEXT: expand-ir-insts<O2>
+; GCN-O2-NEXT: amdgpu-remove-incompatible-functions
+; GCN-O2-NEXT: amdgpu-printf-runtime-binding
+; GCN-O2-NEXT: amdgpu-lower-ctor-dtor
+; GCN-O2-NEXT: function
+; GCN-O2-NEXT: amdgpu-image-intrinsic-opt
+; GCN-O2-NEXT: amdgpu-uniform-intrinsic-combine
+; GCN-O2-NEXT: expand-variadics
+; GCN-O2-NEXT: amdgpu-always-inline
+; GCN-O2-NEXT: always-inline
+; GCN-O2-NEXT: amdgpu-export-kernel-runtime-handles
+; GCN-O2-NEXT: amdgpu-lower-exec-sync
+; GCN-O2-NEXT: amdgpu-sw-lower-lds
+; GCN-O2-NEXT: amdgpu-lower-module-lds
+; GCN-O2-NEXT: function
+; GCN-O2-NEXT: amdgpu-atomic-optimizer
+; GCN-O2-NEXT: atomic-expand
+; GCN-O2-NEXT: amdgpu-promote-alloca
+; GCN-O2-NEXT: separate-const-offset-from-gep<>
+; GCN-O2-NEXT: slsr
+; GCN-O2-NEXT: early-cse<>
+; GCN-O2-NEXT: nary-reassociate
+; GCN-O2-NEXT: early-cse<>
+; GCN-O2-NEXT: amdgpu-codegenprepare
+; GCN-O2-NEXT: loop-mssa
+; GCN-O2-NEXT: licm<allowspeculation>
+; GCN-O2-NEXT: verify
+; GCN-O2-NEXT: loop
+; GCN-O2-NEXT: canon-freeze
+; GCN-O2-NEXT: loop-reduce
+; GCN-O2-NEXT: unreachableblockelim
+; GCN-O2-NEXT: consthoist
+; GCN-O2-NEXT: replace-with-veclib
+; GCN-O2-NEXT: partially-inline-libcalls
+; GCN-O2-NEXT: ee-instrument<post-inline>
+; GCN-O2-NEXT: scalarize-masked-mem-intrin
+; GCN-O2-NEXT: expand-reductions
+; GCN-O2-NEXT: early-cse<>
+; GCN-O2-NEXT: amdgpu-preload-kernel-arguments
+; GCN-O2-NEXT: function
+; GCN-O2-NEXT: amdgpu-lower-kernel-arguments
+; GCN-O2-NEXT: codegenprepare
+; GCN-O2-NEXT: load-store-vectorizer
+; GCN-O2-NEXT: amdgpu-lower-buffer-fat-pointers
+; GCN-O2-NEXT: amdgpu-lower-intrinsics
+; GCN-O2-NEXT: cgscc
+; GCN-O2-NEXT: function
+; GCN-O2-NEXT: lower-switch
+; GCN-O2-NEXT: lower-invoke
+; GCN-O2-NEXT: unreachableblockelim
+; GCN-O2-NEXT: flatten-cfg
+; GCN-O2-NEXT: sink
+; GCN-O2-NEXT: amdgpu-late-codegenprepare
+; GCN-O2-NEXT: amdgpu-unify-divergent-exit-nodes
+; GCN-O2-NEXT: fix-irreducible
+; GCN-O2-NEXT: unify-loop-exits
+; GCN-O2-NEXT: StructurizeCFGPass
+; GCN-O2-NEXT: amdgpu-annotate-uniform
+; GCN-O2-NEXT: si-annotate-control-flow
+; GCN-O2-NEXT: amdgpu-rewrite-undef-for-phi
+; GCN-O2-NEXT: lcssa
+; GCN-O2-NEXT: amdgpu-perf-hint
+; GCN-O2-NEXT: cgscc
+; GCN-O2-NEXT: function
+; GCN-O2-NEXT: inline-asm-prepare
+; GCN-O2-NEXT: safe-stack
+; GCN-O2-NEXT: stack-protector
+; GCN-O2-NEXT: verify
+; GCN-O2-NEXT: amdgpu-asm-printer-begin
+; GCN-O2-NEXT: cgscc
+; GCN-O2-NEXT: function
+; GCN-O2-NEXT: machine-function
+; GCN-O2-NEXT: amdgpu-isel
+; GCN-O2-NEXT: si-fix-sgpr-copies
+; GCN-O2-NEXT: si-i1-copies
+; GCN-O2-NEXT: finalize-isel
+; GCN-O2-NEXT: early-tailduplication
+; GCN-O2-NEXT: opt-phis
+; GCN-O2-NEXT: stack-coloring
+; GCN-O2-NEXT: localstackalloc
+; GCN-O2-NEXT: dead-mi-elimination
+; GCN-O2-NEXT: early-machinelicm
+; GCN-O2-NEXT: machine-cse
+; GCN-O2-NEXT: machine-sink
+; GCN-O2-NEXT: peephole-opt
+; GCN-O2-NEXT: dead-mi-elimination
+; GCN-O2-NEXT: si-fold-operands
+; GCN-O2-NEXT: gcn-dpp-combine
+; GCN-O2-NEXT: si-load-store-opt
+; GCN-O2-NEXT: si-peephole-sdwa
+; GCN-O2-NEXT: early-machinelicm
+; GCN-O2-NEXT: machine-cse
+; GCN-O2-NEXT: si-fold-operands
+; GCN-O2-NEXT: dead-mi-elimination
+; GCN-O2-NEXT: si-shrink-instructions
+; GCN-O2-NEXT: require<reg-usage>
+; GCN-O2-NEXT: cgscc
+; GCN-O2-NEXT: function
+; GCN-O2-NEXT: machine-function
+; GCN-O2-NEXT: reg-usage-propagation
+; GCN-O2-NEXT: amdgpu-prepare-agpr-alloc
+; GCN-O2-NEXT: detect-dead-lanes
+; GCN-O2-NEXT: dead-mi-elimination
+; GCN-O2-NEXT: init-undef
+; GCN-O2-NEXT: process-imp-defs
+; GCN-O2-NEXT: unreachable-mbb-elimination
+; GCN-O2-NEXT: require<live-vars>
+; GCN-O2-NEXT: si-opt-vgpr-liverange
+; GCN-O2-NEXT: require<machine-loops>
+; GCN-O2-NEXT: phi-node-elimination
+; GCN-O2-NEXT: si-lower-control-flow
+; GCN-O2-NEXT: two-address-instruction
+; GCN-O2-NEXT: register-coalescer
+; GCN-O2-NEXT: rename-independent-subregs
+; GCN-O2-NEXT: amdgpu-rewrite-partial-reg-uses
+; GCN-O2-NEXT: machine-scheduler
+; GCN-O2-NEXT: amdgpu-pre-ra-optimizations
+; GCN-O2-NEXT: si-wqm
+; GCN-O2-NEXT: si-optimize-exec-masking-pre-ra
+; GCN-O2-NEXT: si-form-memory-clauses
+; GCN-O2-NEXT: amdgpu-pre-ra-long-branch-reg
+; GCN-O2-NEXT: greedy<sgpr>
+; GCN-O2-NEXT: virt-reg-rewriter<no-clear-vregs>
+; GCN-O2-NEXT: stack-slot-coloring
+; GCN-O2-NEXT: si-lower-sgpr-spills
+; GCN-O2-NEXT: si-pre-allocate-wwm-regs
+; GCN-O2-NEXT: greedy<wwm>
+; GCN-O2-NEXT: si-lower-wwm-copies
+; GCN-O2-NEXT: virt-reg-rewriter<no-clear-vregs>
+; GCN-O2-NEXT: amdgpu-reserve-wwm-regs
+; GCN-O2-NEXT: greedy<vgpr>
+; GCN-O2-NEXT: amdgpu-nsa-reassign
+; GCN-O2-NEXT: amdgpu-rewrite-agpr-copy-mfma
+; GCN-O2-NEXT: virt-reg-rewriter
+; GCN-O2-NEXT: amdgpu-mark-last-scratch-load
+; GCN-O2-NEXT: stack-slot-coloring
+; GCN-O2-NEXT: machine-cp
+; GCN-O2-NEXT: machinelicm
+; GCN-O2-NEXT: si-fix-vgpr-copies
+; GCN-O2-NEXT: si-optimize-exec-masking
+; GCN-O2-NEXT: remove-redundant-debug-values
+; GCN-O2-NEXT: fixup-statepoint-caller-saved
+; GCN-O2-NEXT: postra-machine-sink
+; GCN-O2-NEXT: shrink-wrap
+; GCN-O2-NEXT: prolog-epilog
+; GCN-O2-NEXT: machine-latecleanup
+; GCN-O2-NEXT: branch-folder
+; GCN-O2-NEXT: tailduplication
+; GCN-O2-NEXT: machine-cp
+; GCN-O2-NEXT: post-ra-pseudos
+; GCN-O2-NEXT: si-shrink-instructions
+; GCN-O2-NEXT: si-post-ra-bundler
+; GCN-O2-NEXT: postmisched
+; GCN-O2-NEXT: block-placement
+; GCN-O2-NEXT: fentry-insert
+; GCN-O2-NEXT: xray-instrumentation
+; GCN-O2-NEXT: gcn-create-vopd
+; GCN-O2-NEXT: si-memory-legalizer
+; GCN-O2-NEXT: si-post-ra-16bit-mov-folding
+; GCN-O2-NEXT: si-insert-waitcnts
+; GCN-O2-NEXT: si-mode-register
+; GCN-O2-NEXT: si-insert-hard-clauses
+; GCN-O2-NEXT: si-late-branch-lowering
+; GCN-O2-NEXT: si-pre-emit-peephole
+; GCN-O2-NEXT: post-RA-hazard-rec
+; GCN-O2-NEXT: amdgpu-wait-sgpr-hazards
+; GCN-O2-NEXT: amdgpu-lower-vgpr-encoding
+; GCN-O2-NEXT: amdgpu-insert-delay-alu
+; GCN-O2-NEXT: branch-relaxation
+; GCN-O2-NEXT: reg-usage-collector
+; GCN-O2-NEXT: remove-loads-into-fake-uses
+; GCN-O2-NEXT: live-debug-values
+; GCN-O2-NEXT: machine-sanmd
+; GCN-O2-NEXT: amdgpu-preload-kern-arg-prolog
+; GCN-O2-NEXT: stack-frame-layout
+; GCN-O2-NEXT: amdgpu-asm-printer
+; GCN-O2-NEXT: free-machine-function
+; GCN-O2-NEXT: amdgpu-asm-printer-end
-; RUN: llc -O3 -enable-new-pm -mtriple=amdgpu7.00--amdhsa -global-isel=1 -print-pipeline-passes=tree < %s 2>&1 \
-; RUN: | FileCheck -check-prefix=GCN-O3-GISEL %s
-
-; GCN-O0-SDAG: require<MachineModuleAnalysis>
-; GCN-O0-SDAG-NEXT: require<profile-summary>
-; GCN-O0-SDAG-NEXT: require<collector-metadata>
-; GCN-O0-SDAG-NEXT: require<runtime-libcall-info>
-; GCN-O0-SDAG-NEXT: require<libcall-lowering-info>
-; GCN-O0-SDAG-NEXT: pre-isel-intrinsic-lowering
-; GCN-O0-SDAG-NEXT: function
-; GCN-O0-SDAG-NEXT: expand-ir-insts<O0>
-; GCN-O0-SDAG-NEXT: amdgpu-remove-incompatible-functions
-; GCN-O0-SDAG-NEXT: amdgpu-printf-runtime-binding
-; GCN-O0-SDAG-NEXT: amdgpu-lower-ctor-dtor
-; GCN-O0-SDAG-NEXT: function
-; GCN-O0-SDAG-NEXT: amdgpu-uniform-intrinsic-combine
-; GCN-O0-SDAG-NEXT: expand-variadics
-; GCN-O0-SDAG-NEXT: amdgpu-always-inline
-; GCN-O0-SDAG-NEXT: always-inline
-; GCN-O0-SDAG-NEXT: amdgpu-export-kernel-runtime-handles
-; GCN-O0-SDAG-NEXT: amdgpu-lower-exec-sync
-; GCN-O0-SDAG-NEXT: amdgpu-sw-lower-lds
-; GCN-O0-SDAG-NEXT: amdgpu-lower-module-lds
-; GCN-O0-SDAG-NEXT: function
-; GCN-O0-SDAG-NEXT: atomic-expand
-; GCN-O0-SDAG-NEXT: verify
-; GCN-O0-SDAG-NEXT: unreachableblockelim
-; GCN-O0-SDAG-NEXT: ee-instrument<post-inline>
-; GCN-O0-SDAG-NEXT: scalarize-masked-mem-intrin
-; GCN-O0-SDAG-NEXT: expand-reductions
-; GCN-O0-SDAG-NEXT: amdgpu-lower-kernel-arguments
-; GCN-O0-SDAG-NEXT: amdgpu-lower-buffer-fat-pointers
-; GCN-O0-SDAG-NEXT: amdgpu-lower-intrinsics
-; GCN-O0-SDAG-NEXT: cgscc
-; GCN-O0-SDAG-NEXT: function
-; GCN-O0-SDAG-NEXT: lower-switch
-; GCN-O0-SDAG-NEXT: lower-invoke
-; GCN-O0-SDAG-NEXT: unreachableblockelim
-; GCN-O0-SDAG-NEXT: amdgpu-unify-divergent-exit-nodes
-; GCN-O0-SDAG-NEXT: fix-irreducible
-; GCN-O0-SDAG-NEXT: unify-loop-exits
-; GCN-O0-SDAG-NEXT: StructurizeCFGPass
-; GCN-O0-SDAG-NEXT: amdgpu-annotate-uniform
-; GCN-O0-SDAG-NEXT: si-annotate-control-flow
-; GCN-O0-SDAG-NEXT: amdgpu-rewrite-undef-for-phi
-; GCN-O0-SDAG-NEXT: lcssa
-; GCN-O0-SDAG-NEXT: inline-asm-prepare
-; GCN-O0-SDAG-NEXT: safe-stack
-; GCN-O0-SDAG-NEXT: stack-protector
-; GCN-O0-SDAG-NEXT: verify
-; GCN-O0-SDAG-NEXT: amdgpu-asm-printer-begin
-; GCN-O0-SDAG-NEXT: cgscc
-; GCN-O0-SDAG-NEXT: function
-; GCN-O0-SDAG-NEXT: machine-function
-; GCN-O0-SDAG-NEXT: amdgpu-isel
-; GCN-O0-SDAG-NEXT: si-fix-sgpr-copies
-; GCN-O0-SDAG-NEXT: si-i1-copies
-; GCN-O0-SDAG-NEXT: finalize-isel
-; GCN-O0-SDAG-NEXT: localstackalloc
-; GCN-O0-SDAG-NEXT: require<reg-usage>
-; GCN-O0-SDAG-NEXT: cgscc
-; GCN-O0-SDAG-NEXT: function
-; GCN-O0-SDAG-NEXT: machine-function
-; GCN-O0-SDAG-NEXT: reg-usage-propagation
-; GCN-O0-SDAG-NEXT: phi-node-elimination
-; GCN-O0-SDAG-NEXT: si-lower-control-flow
-; GCN-O0-SDAG-NEXT: two-address-instruction
-; GCN-O0-SDAG-NEXT: si-wqm
-; GCN-O0-SDAG-NEXT: amdgpu-pre-ra-long-branch-reg
-; GCN-O0-SDAG-NEXT: regallocfast<filter=sgpr;no-clear-vregs>
-; GCN-O0-SDAG-NEXT: si-lower-sgpr-spills
-; GCN-O0-SDAG-NEXT: si-pre-allocate-wwm-regs
-; GCN-O0-SDAG-NEXT: regallocfast<filter=wwm;no-clear-vregs>
-; GCN-O0-SDAG-NEXT: si-lower-wwm-copies
-; GCN-O0-SDAG-NEXT: amdgpu-reserve-wwm-regs
-; GCN-O0-SDAG-NEXT: regallocfast<filter=vgpr>
-; GCN-O0-SDAG-NEXT: si-fix-vgpr-copies
-; GCN-O0-SDAG-NEXT: remove-redundant-debug-values
-; GCN-O0-SDAG-NEXT: fixup-statepoint-caller-saved
-; GCN-O0-SDAG-NEXT: prolog-epilog
-; GCN-O0-SDAG-NEXT: post-ra-pseudos
-; GCN-O0-SDAG-NEXT: si-post-ra-bundler
-; GCN-O0-SDAG-NEXT: fentry-insert
-; GCN-O0-SDAG-NEXT: xray-instrumentation
-; GCN-O0-SDAG-NEXT: si-memory-legalizer
-; GCN-O0-SDAG-NEXT: si-insert-waitcnts
-; GCN-O0-SDAG-NEXT: si-mode-register
-; GCN-O0-SDAG-NEXT: si-late-branch-lowering
-; GCN-O0-SDAG-NEXT: post-RA-hazard-rec
-; GCN-O0-SDAG-NEXT: amdgpu-wait-sgpr-hazards
-; GCN-O0-SDAG-NEXT: amdgpu-lower-vgpr-encoding
-; GCN-O0-SDAG-NEXT: branch-relaxation
-; GCN-O0-SDAG-NEXT: reg-usage-collector
-; GCN-O0-SDAG-NEXT: remove-loads-into-fake-uses
-; GCN-O0-SDAG-NEXT: live-debug-values
-; GCN-O0-SDAG-NEXT: machine-sanmd
-; GCN-O0-SDAG-NEXT: amdgpu-preload-kern-arg-prolog
-; GCN-O0-SDAG-NEXT: stack-frame-layout
-; GCN-O0-SDAG-NEXT: amdgpu-asm-printer
-; GCN-O0-SDAG-NEXT: free-machine-function
-; GCN-O0-SDAG-NEXT: amdgpu-asm-printer-end
-
-; GCN-O0-GISEL: require<MachineModuleAnalysis>
-; GCN-O0-GISEL-NEXT: require<profile-summary>
-; GCN-O0-GISEL-NEXT: require<collector-metadata>
-; GCN-O0-GISEL-NEXT: require<runtime-libcall-info>
-; GCN-O0-GISEL-NEXT: require<libcall-lowering-info>
-; GCN-O0-GISEL-NEXT: pre-isel-intrinsic-lowering
-; GCN-O0-GISEL-NEXT: function
-; GCN-O0-GISEL-NEXT: expand-ir-insts<O0>
-; GCN-O0-GISEL-NEXT: amdgpu-remove-incompatible-functions
-; GCN-O0-GISEL-NEXT: amdgpu-printf-runtime-binding
-; GCN-O0-GISEL-NEXT: amdgpu-lower-ctor-dtor
-; GCN-O0-GISEL-NEXT: function
-; GCN-O0-GISEL-NEXT: amdgpu-uniform-intrinsic-combine
-; GCN-O0-GISEL-NEXT: expand-variadics
-; GCN-O0-GISEL-NEXT: amdgpu-always-inline
-; GCN-O0-GISEL-NEXT: always-inline
-; GCN-O0-GISEL-NEXT: amdgpu-export-kernel-runtime-handles
-; GCN-O0-GISEL-NEXT: amdgpu-lower-exec-sync
-; GCN-O0-GISEL-NEXT: amdgpu-sw-lower-lds
-; GCN-O0-GISEL-NEXT: amdgpu-lower-module-lds
-; GCN-O0-GISEL-NEXT: function
-; GCN-O0-GISEL-NEXT: atomic-expand
-; GCN-O0-GISEL-NEXT: verify
-; GCN-O0-GISEL-NEXT: unreachableblockelim
-; GCN-O0-GISEL-NEXT: ee-instrument<post-inline>
-; GCN-O0-GISEL-NEXT: scalarize-masked-mem-intrin
-; GCN-O0-GISEL-NEXT: expand-reductions
-; GCN-O0-GISEL-NEXT: amdgpu-lower-kernel-arguments
-; GCN-O0-GISEL-NEXT: amdgpu-lower-buffer-fat-pointers
-; GCN-O0-GISEL-NEXT: amdgpu-lower-intrinsics
-; GCN-O0-GISEL-NEXT: cgscc
-; GCN-O0-GISEL-NEXT: function
-; GCN-O0-GISEL-NEXT: lower-switch
-; GCN-O0-GISEL-NEXT: lower-invoke
-; GCN-O0-GISEL-NEXT: unreachableblockelim
-; GCN-O0-GISEL-NEXT: amdgpu-unify-divergent-exit-nodes
-; GCN-O0-GISEL-NEXT: fix-irreducible
-; GCN-O0-GISEL-NEXT: unify-loop-exits
-; GCN-O0-GISEL-NEXT: StructurizeCFGPass
-; GCN-O0-GISEL-NEXT: amdgpu-annotate-uniform
-; GCN-O0-GISEL-NEXT: si-annotate-control-flow
-; GCN-O0-GISEL-NEXT: amdgpu-rewrite-undef-for-phi
-; GCN-O0-GISEL-NEXT: inline-asm-prepare
-; GCN-O0-GISEL-NEXT: safe-stack
-; GCN-O0-GISEL-NEXT: stack-protector
-; GCN-O0-GISEL-NEXT: verify
-; GCN-O0-GISEL-NEXT: amdgpu-asm-printer-begin
-; GCN-O0-GISEL-NEXT: cgscc
-; GCN-O0-GISEL-NEXT: function
-; GCN-O0-GISEL-NEXT: machine-function
-; GCN-O0-GISEL-NEXT: ir-translator
-; GCN-O0-GISEL-NEXT: amdgpu-prelegalizer-combiner
-; GCN-O0-GISEL-NEXT: localizer
-; GCN-O0-GISEL-NEXT: legalizer
-; GCN-O0-GISEL-NEXT: amdgpu-postlegalizer-combiner
-; GCN-O0-GISEL-NEXT: amdgpu-global-isel-divergence-lowering
-; GCN-O0-GISEL-NEXT: amdgpu-reg-bank-select
-; GCN-O0-GISEL-NEXT: amdgpu-reg-bank-legalize
-; GCN-O0-GISEL-NEXT: amdgpu-regbank-combiner
-; GCN-O0-GISEL-NEXT: instruction-select
-; GCN-O0-GISEL-NEXT: reset-machine-function
-; GCN-O0-GISEL-NEXT: finalize-isel
-; GCN-O0-GISEL-NEXT: localstackalloc
-; GCN-O0-GISEL-NEXT: require<reg-usage>
-; GCN-O0-GISEL-NEXT: cgscc
-; GCN-O0-GISEL-NEXT: function
-; GCN-O0-GISEL-NEXT: machine-function
-; GCN-O0-GISEL-NEXT: reg-usage-propagation
-; GCN-O0-GISEL-NEXT: phi-node-elimination
-; GCN-O0-GISEL-NEXT: si-lower-control-flow
-; GCN-O0-GISEL-NEXT: two-address-instruction
-; GCN-O0-GISEL-NEXT: si-wqm
-; GCN-O0-GISEL-NEXT: amdgpu-pre-ra-long-branch-reg
-; GCN-O0-GISEL-NEXT: regallocfast<filter=sgpr;no-clear-vregs>
-; GCN-O0-GISEL-NEXT: si-lower-sgpr-spills
-; GCN-O0-GISEL-NEXT: si-pre-allocate-wwm-regs
-; GCN-O0-GISEL-NEXT: regallocfast<filter=wwm;no-clear-vregs>
-; GCN-O0-GISEL-NEXT: si-lower-wwm-copies
-; GCN-O0-GISEL-NEXT: amdgpu-reserve-wwm-regs
-; GCN-O0-GISEL-NEXT: regallocfast<filter=vgpr>
-; GCN-O0-GISEL-NEXT: si-fix-vgpr-copies
-; GCN-O0-GISEL-NEXT: remove-redundant-debug-values
-; GCN-O0-GISEL-NEXT: fixup-statepoint-caller-saved
-; GCN-O0-GISEL-NEXT: prolog-epilog
-; GCN-O0-GISEL-NEXT: post-ra-pseudos
-; GCN-O0-GISEL-NEXT: si-post-ra-bundler
-; GCN-O0-GISEL-NEXT: fentry-insert
-; GCN-O0-GISEL-NEXT: xray-instrumentation
-; GCN-O0-GISEL-NEXT: si-memory-legalizer
-; GCN-O0-GISEL-NEXT: si-insert-waitcnts
-; GCN-O0-GISEL-NEXT: si-mode-register
-; GCN-O0-GISEL-NEXT: si-late-branch-lowering
-; GCN-O0-GISEL-NEXT: post-RA-hazard-rec
-; GCN-O0-GISEL-NEXT: amdgpu-wait-sgpr-hazards
-; GCN-O0-GISEL-NEXT: amdgpu-lower-vgpr-encoding
-; GCN-O0-GISEL-NEXT: branch-relaxation
-; GCN-O0-GISEL-NEXT: reg-usage-collector
-; GCN-O0-GISEL-NEXT: remove-loads-into-fake-uses
-; GCN-O0-GISEL-NEXT: live-debug-values
-; GCN-O0-GISEL-NEXT: machine-sanmd
-; GCN-O0-GISEL-NEXT: amdgpu-preload-kern-arg-prolog
-; GCN-O0-GISEL-NEXT: stack-frame-layout
-; GCN-O0-GISEL-NEXT: amdgpu-asm-printer
-; GCN-O0-GISEL-NEXT: free-machine-function
-; GCN-O0-GISEL-NEXT: amdgpu-asm-printer-end
-
-; GCN-O2-SDAG: require<MachineModuleAnalysis>
-; GCN-O2-SDAG-NEXT: require<profile-summary>
-; GCN-O2-SDAG-NEXT: require<collector-metadata>
-; GCN-O2-SDAG-NEXT: require<runtime-libcall-info>
-; GCN-O2-SDAG-NEXT: require<libcall-lowering-info>
-; GCN-O2-SDAG-NEXT: function
-; GCN-O2-SDAG-NEXT: objc-arc-contract
-; GCN-O2-SDAG-NEXT: pre-isel-intrinsic-lowering
-; GCN-O2-SDAG-NEXT: function
-; GCN-O2-SDAG-NEXT: expand-ir-insts<O2>
-; GCN-O2-SDAG-NEXT: amdgpu-remove-incompatible-functions
-; GCN-O2-SDAG-NEXT: amdgpu-printf-runtime-binding
-; GCN-O2-SDAG-NEXT: amdgpu-lower-ctor-dtor
-; GCN-O2-SDAG-NEXT: function
-; GCN-O2-SDAG-NEXT: amdgpu-image-intrinsic-opt
-; GCN-O2-SDAG-NEXT: amdgpu-uniform-intrinsic-combine
-; GCN-O2-SDAG-NEXT: expand-variadics
-; GCN-O2-SDAG-NEXT: amdgpu-always-inline
-; GCN-O2-SDAG-NEXT: always-inline
-; GCN-O2-SDAG-NEXT: amdgpu-export-kernel-runtime-handles
-; GCN-O2-SDAG-NEXT: amdgpu-lower-exec-sync
-; GCN-O2-SDAG-NEXT: amdgpu-sw-lower-lds
-; GCN-O2-SDAG-NEXT: amdgpu-lower-module-lds
-; GCN-O2-SDAG-NEXT: function
-; GCN-O2-SDAG-NEXT: amdgpu-atomic-optimizer
-; GCN-O2-SDAG-NEXT: atomic-expand
-; GCN-O2-SDAG-NEXT: amdgpu-promote-alloca
-; GCN-O2-SDAG-NEXT: separate-const-offset-from-gep<>
-; GCN-O2-SDAG-NEXT: slsr
-; GCN-O2-SDAG-NEXT: early-cse<>
-; GCN-O2-SDAG-NEXT: nary-reassociate
-; GCN-O2-SDAG-NEXT: early-cse<>
-; GCN-O2-SDAG-NEXT: amdgpu-codegenprepare
-; GCN-O2-SDAG-NEXT: loop-mssa
-; GCN-O2-SDAG-NEXT: licm<allowspeculation>
-; GCN-O2-SDAG-NEXT: verify
-; GCN-O2-SDAG-NEXT: loop
-; GCN-O2-SDAG-NEXT: canon-freeze
-; GCN-O2-SDAG-NEXT: loop-reduce
-; GCN-O2-SDAG-NEXT: unreachableblockelim
-; GCN-O2-SDAG-NEXT: consthoist
-; GCN-O2-SDAG-NEXT: replace-with-veclib
-; GCN-O2-SDAG-NEXT: partially-inline-libcalls
-; GCN-O2-SDAG-NEXT: ee-instrument<post-inline>
-; GCN-O2-SDAG-NEXT: scalarize-masked-mem-intrin
-; GCN-O2-SDAG-NEXT: expand-reductions
-; GCN-O2-SDAG-NEXT: early-cse<>
-; GCN-O2-SDAG-NEXT: amdgpu-preload-kernel-arguments
-; GCN-O2-SDAG-NEXT: function
-; GCN-O2-SDAG-NEXT: amdgpu-lower-kernel-arguments
-; GCN-O2-SDAG-NEXT: codegenprepare
-; GCN-O2-SDAG-NEXT: load-store-vectorizer
-; GCN-O2-SDAG-NEXT: amdgpu-lower-buffer-fat-pointers
-; GCN-O2-SDAG-NEXT: amdgpu-lower-intrinsics
-; GCN-O2-SDAG-NEXT: cgscc
-; GCN-O2-SDAG-NEXT: function
-; GCN-O2-SDAG-NEXT: lower-switch
-; GCN-O2-SDAG-NEXT: lower-invoke
-; GCN-O2-SDAG-NEXT: unreachableblockelim
-; GCN-O2-SDAG-NEXT: flatten-cfg
-; GCN-O2-SDAG-NEXT: sink
-; GCN-O2-SDAG-NEXT: amdgpu-late-codegenprepare
-; GCN-O2-SDAG-NEXT: amdgpu-unify-divergent-exit-nodes
-; GCN-O2-SDAG-NEXT: fix-irreducible
-; GCN-O2-SDAG-NEXT: unify-loop-exits
-; GCN-O2-SDAG-NEXT: StructurizeCFGPass
-; GCN-O2-SDAG-NEXT: amdgpu-annotate-uniform
-; GCN-O2-SDAG-NEXT: si-annotate-control-flow
-; GCN-O2-SDAG-NEXT: amdgpu-rewrite-undef-for-phi
-; GCN-O2-SDAG-NEXT: lcssa
-; GCN-O2-SDAG-NEXT: amdgpu-perf-hint
-; GCN-O2-SDAG-NEXT: cgscc
-; GCN-O2-SDAG-NEXT: function
-; GCN-O2-SDAG-NEXT: inline-asm-prepare
-; GCN-O2-SDAG-NEXT: safe-stack
-; GCN-O2-SDAG-NEXT: stack-protector
-; GCN-O2-SDAG-NEXT: verify
-; GCN-O2-SDAG-NEXT: amdgpu-asm-printer-begin
-; GCN-O2-SDAG-NEXT: cgscc
-; GCN-O2-SDAG-NEXT: function
-; GCN-O2-SDAG-NEXT: machine-function
-; GCN-O2-SDAG-NEXT: amdgpu-isel
-; GCN-O2-SDAG-NEXT: si-fix-sgpr-copies
-; GCN-O2-SDAG-NEXT: si-i1-copies
-; GCN-O2-SDAG-NEXT: finalize-isel
-; GCN-O2-SDAG-NEXT: early-tailduplication
-; GCN-O2-SDAG-NEXT: opt-phis
-; GCN-O2-SDAG-NEXT: stack-coloring
-; GCN-O2-SDAG-NEXT: localstackalloc
-; GCN-O2-SDAG-NEXT: dead-mi-elimination
-; GCN-O2-SDAG-NEXT: early-machinelicm
-; GCN-O2-SDAG-NEXT: machine-cse
-; GCN-O2-SDAG-NEXT: machine-sink
-; GCN-O2-SDAG-NEXT: peephole-opt
-; GCN-O2-SDAG-NEXT: dead-mi-elimination
-; GCN-O2-SDAG-NEXT: si-fold-operands
-; GCN-O2-SDAG-NEXT: gcn-dpp-combine
-; GCN-O2-SDAG-NEXT: si-load-store-opt
-; GCN-O2-SDAG-NEXT: si-peephole-sdwa
-; GCN-O2-SDAG-NEXT: early-machinelicm
-; GCN-O2-SDAG-NEXT: machine-cse
-; GCN-O2-SDAG-NEXT: si-fold-operands
-; GCN-O2-SDAG-NEXT: dead-mi-elimination
-; GCN-O2-SDAG-NEXT: si-shrink-instructions
-; GCN-O2-SDAG-NEXT: require<reg-usage>
-; GCN-O2-SDAG-NEXT: cgscc
-; GCN-O2-SDAG-NEXT: function
-; GCN-O2-SDAG-NEXT: machine-function
-; GCN-O2-SDAG-NEXT: reg-usage-propagation
-; GCN-O2-SDAG-NEXT: amdgpu-prepare-agpr-alloc
-; GCN-O2-SDAG-NEXT: detect-dead-lanes
-; GCN-O2-SDAG-NEXT: dead-mi-elimination
-; GCN-O2-SDAG-NEXT: init-undef
-; GCN-O2-SDAG-NEXT: process-imp-defs
-; GCN-O2-SDAG-NEXT: unreachable-mbb-elimination
-; GCN-O2-SDAG-NEXT: require<live-vars>
-; GCN-O2-SDAG-NEXT: si-opt-vgpr-liverange
-; GCN-O2-SDAG-NEXT: require<machine-loops>
-; GCN-O2-SDAG-NEXT: phi-node-elimination
-; GCN-O2-SDAG-NEXT: si-lower-control-flow
-; GCN-O2-SDAG-NEXT: two-address-instruction
-; GCN-O2-SDAG-NEXT: register-coalescer
-; GCN-O2-SDAG-NEXT: rename-independent-subregs
-; GCN-O2-SDAG-NEXT: amdgpu-rewrite-partial-reg-uses
-; GCN-O2-SDAG-NEXT: machine-scheduler
-; GCN-O2-SDAG-NEXT: amdgpu-pre-ra-optimizations
-; GCN-O2-SDAG-NEXT: si-wqm
-; GCN-O2-SDAG-NEXT: si-optimize-exec-masking-pre-ra
-; GCN-O2-SDAG-NEXT: si-form-memory-clauses
-; GCN-O2-SDAG-NEXT: amdgpu-pre-ra-long-branch-reg
-; GCN-O2-SDAG-NEXT: greedy<sgpr>
-; GCN-O2-SDAG-NEXT: virt-reg-rewriter<no-clear-vregs>
-; GCN-O2-SDAG-NEXT: stack-slot-coloring
-; GCN-O2-SDAG-NEXT: si-lower-sgpr-spills
-; GCN-O2-SDAG-NEXT: si-pre-allocate-wwm-regs
-; GCN-O2-SDAG-NEXT: greedy<wwm>
-; GCN-O2-SDAG-NEXT: si-lower-wwm-copies
-; GCN-O2-SDAG-NEXT: virt-reg-rewriter<no-clear-vregs>
-; GCN-O2-SDAG-NEXT: amdgpu-reserve-wwm-regs
-; GCN-O2-SDAG-NEXT: greedy<vgpr>
-; GCN-O2-SDAG-NEXT: amdgpu-nsa-reassign
-; GCN-O2-SDAG-NEXT: amdgpu-rewrite-agpr-copy-mfma
-; GCN-O2-SDAG-NEXT: virt-reg-rewriter
-; GCN-O2-SDAG-NEXT: amdgpu-mark-last-scratch-load
-; GCN-O2-SDAG-NEXT: stack-slot-coloring
-; GCN-O2-SDAG-NEXT: machine-cp
-; GCN-O2-SDAG-NEXT: machinelicm
-; GCN-O2-SDAG-NEXT: si-fix-vgpr-copies
-; GCN-O2-SDAG-NEXT: si-optimize-exec-masking
-; GCN-O2-SDAG-NEXT: remove-redundant-debug-values
-; GCN-O2-SDAG-NEXT: fixup-statepoint-caller-saved
-; GCN-O2-SDAG-NEXT: postra-machine-sink
-; GCN-O2-SDAG-NEXT: shrink-wrap
-; GCN-O2-SDAG-NEXT: prolog-epilog
-; GCN-O2-SDAG-NEXT: machine-latecleanup
-; GCN-O2-SDAG-NEXT: branch-folder
-; GCN-O2-SDAG-NEXT: tailduplication
-; GCN-O2-SDAG-NEXT: machine-cp
-; GCN-O2-SDAG-NEXT: post-ra-pseudos
-; GCN-O2-SDAG-NEXT: si-shrink-instructions
-; GCN-O2-SDAG-NEXT: si-post-ra-bundler
-; GCN-O2-SDAG-NEXT: postmisched
-; GCN-O2-SDAG-NEXT: block-placement
-; GCN-O2-SDAG-NEXT: fentry-insert
-; GCN-O2-SDAG-NEXT: xray-instrumentation
-; GCN-O2-SDAG-NEXT: gcn-create-vopd
-; GCN-O2-SDAG-NEXT: si-memory-legalizer
-; GCN-O2-SDAG-NEXT: si-insert-waitcnts
-; GCN-O2-SDAG-NEXT: si-mode-register
-; GCN-O2-SDAG-NEXT: si-insert-hard-clauses
-; GCN-O2-SDAG-NEXT: si-late-branch-lowering
-; GCN-O2-SDAG-NEXT: si-pre-emit-peephole
-; GCN-O2-SDAG-NEXT: post-RA-hazard-rec
-; GCN-O2-SDAG-NEXT: amdgpu-wait-sgpr-hazards
-; GCN-O2-SDAG-NEXT: amdgpu-lower-vgpr-encoding
-; GCN-O2-SDAG-NEXT: amdgpu-insert-delay-alu
-; GCN-O2-SDAG-NEXT: branch-relaxation
-; GCN-O2-SDAG-NEXT: reg-usage-collector
-; GCN-O2-SDAG-NEXT: remove-loads-into-fake-uses
-; GCN-O2-SDAG-NEXT: live-debug-values
-; GCN-O2-SDAG-NEXT: machine-sanmd
-; GCN-O2-SDAG-NEXT: amdgpu-preload-kern-arg-prolog
-; GCN-O2-SDAG-NEXT: stack-frame-layout
-; GCN-O2-SDAG-NEXT: amdgpu-asm-printer
-; GCN-O2-SDAG-NEXT: free-machine-function
-; GCN-O2-SDAG-NEXT: amdgpu-asm-printer-end
-
-; GCN-O2-GISEL: require<MachineModuleAnalysis>
-; GCN-O2-GISEL-NEXT: require<profile-summary>
-; GCN-O2-GISEL-NEXT: require<collector-metadata>
-; GCN-O2-GISEL-NEXT: require<runtime-libcall-info>
-; GCN-O2-GISEL-NEXT: require<libcall-lowering-info>
-; GCN-O2-GISEL-NEXT: function
-; GCN-O2-GISEL-NEXT: objc-arc-contract
-; GCN-O2-GISEL-NEXT: pre-isel-intrinsic-lowering
-; GCN-O2-GISEL-NEXT: function
-; GCN-O2-GISEL-NEXT: expand-ir-insts<O2>
-; GCN-O2-GISEL-NEXT: amdgpu-remove-incompatible-functions
-; GCN-O2-GISEL-NEXT: amdgpu-printf-runtime-binding
-; GCN-O2-GISEL-NEXT: amdgpu-lower-ctor-dtor
-; GCN-O2-GISEL-NEXT: function
-; GCN-O2-GISEL-NEXT: amdgpu-image-intrinsic-opt
-; GCN-O2-GISEL-NEXT: amdgpu-uniform-intrinsic-combine
-; GCN-O2-GISEL-NEXT: expand-variadics
-; GCN-O2-GISEL-NEXT: amdgpu-always-inline
-; GCN-O2-GISEL-NEXT: always-inline
-; GCN-O2-GISEL-NEXT: amdgpu-export-kernel-runtime-handles
-; GCN-O2-GISEL-NEXT: amdgpu-lower-exec-sync
-; GCN-O2-GISEL-NEXT: amdgpu-sw-lower-lds
-; GCN-O2-GISEL-NEXT: amdgpu-lower-module-lds
-; GCN-O2-GISEL-NEXT: function
-; GCN-O2-GISEL-NEXT: amdgpu-atomic-optimizer
-; GCN-O2-GISEL-NEXT: atomic-expand
-; GCN-O2-GISEL-NEXT: amdgpu-promote-alloca
-; GCN-O2-GISEL-NEXT: separate-const-offset-from-gep<>
-; GCN-O2-GISEL-NEXT: slsr
-; GCN-O2-GISEL-NEXT: early-cse<>
-; GCN-O2-GISEL-NEXT: nary-reassociate
-; GCN-O2-GISEL-NEXT: early-cse<>
-; GCN-O2-GISEL-NEXT: amdgpu-codegenprepare
-; GCN-O2-GISEL-NEXT: loop-mssa
-; GCN-O2-GISEL-NEXT: licm<allowspeculation>
-; GCN-O2-GISEL-NEXT: verify
-; GCN-O2-GISEL-NEXT: loop
-; GCN-O2-GISEL-NEXT: canon-freeze
-; GCN-O2-GISEL-NEXT: loop-reduce
-; GCN-O2-GISEL-NEXT: unreachableblockelim
-; GCN-O2-GISEL-NEXT: consthoist
-; GCN-O2-GISEL-NEXT: replace-with-veclib
-; GCN-O2-GISEL-NEXT: partially-inline-libcalls
-; GCN-O2-GISEL-NEXT: ee-instrument<post-inline>
-; GCN-O2-GISEL-NEXT: scalarize-masked-mem-intrin
-; GCN-O2-GISEL-NEXT: expand-reductions
-; GCN-O2-GISEL-NEXT: early-cse<>
-; GCN-O2-GISEL-NEXT: amdgpu-preload-kernel-arguments
-; GCN-O2-GISEL-NEXT: function
-; GCN-O2-GISEL-NEXT: amdgpu-lower-kernel-arguments
-; GCN-O2-GISEL-NEXT: codegenprepare
-; GCN-O2-GISEL-NEXT: load-store-vectorizer
-; GCN-O2-GISEL-NEXT: amdgpu-lower-buffer-fat-pointers
-; GCN-O2-GISEL-NEXT: amdgpu-lower-intrinsics
-; GCN-O2-GISEL-NEXT: cgscc
-; GCN-O2-GISEL-NEXT: function
-; GCN-O2-GISEL-NEXT: lower-switch
-; GCN-O2-GISEL-NEXT: lower-invoke
-; GCN-O2-GISEL-NEXT: unreachableblockelim
-; GCN-O2-GISEL-NEXT: flatten-cfg
-; GCN-O2-GISEL-NEXT: sink
-; GCN-O2-GISEL-NEXT: amdgpu-late-codegenprepare
-; GCN-O2-GISEL-NEXT: amdgpu-unify-divergent-exit-nodes
-; GCN-O2-GISEL-NEXT: fix-irreducible
-; GCN-O2-GISEL-NEXT: unify-loop-exits
-; GCN-O2-GISEL-NEXT: StructurizeCFGPass
-; GCN-O2-GISEL-NEXT: amdgpu-annotate-uniform
-; GCN-O2-GISEL-NEXT: si-annotate-control-flow
-; GCN-O2-GISEL-NEXT: amdgpu-rewrite-undef-for-phi
-; GCN-O2-GISEL-NEXT: amdgpu-perf-hint
-; GCN-O2-GISEL-NEXT: cgscc
-; GCN-O2-GISEL-NEXT: function
-; GCN-O2-GISEL-NEXT: inline-asm-prepare
-; GCN-O2-GISEL-NEXT: safe-stack
-; GCN-O2-GISEL-NEXT: stack-protector
-; GCN-O2-GISEL-NEXT: verify
-; GCN-O2-GISEL-NEXT: amdgpu-asm-printer-begin
-; GCN-O2-GISEL-NEXT: cgscc
-; GCN-O2-GISEL-NEXT: function
-; GCN-O2-GISEL-NEXT: machine-function
-; GCN-O2-GISEL-NEXT: ir-translator
-; GCN-O2-GISEL-NEXT: amdgpu-prelegalizer-combiner
-; GCN-O2-GISEL-NEXT: localizer
-; GCN-O2-GISEL-NEXT: legalizer
-; GCN-O2-GISEL-NEXT: amdgpu-postlegalizer-combiner
-; GCN-O2-GISEL-NEXT: amdgpu-global-isel-divergence-lowering
-; GCN-O2-GISEL-NEXT: amdgpu-reg-bank-select
-; GCN-O2-GISEL-NEXT: amdgpu-reg-bank-legalize
-; GCN-O2-GISEL-NEXT: amdgpu-regbank-combiner
-; GCN-O2-GISEL-NEXT: instruction-select
-; GCN-O2-GISEL-NEXT: reset-machine-function
-; GCN-O2-GISEL-NEXT: finalize-isel
-; GCN-O2-GISEL-NEXT: early-tailduplication
-; GCN-O2-GISEL-NEXT: opt-phis
-; GCN-O2-GISEL-NEXT: stack-coloring
-; GCN-O2-GISEL-NEXT: localstackalloc
-; GCN-O2-GISEL-NEXT: dead-mi-elimination
-; GCN-O2-GISEL-NEXT: early-machinelicm
-; GCN-O2-GISEL-NEXT: machine-cse
-; GCN-O2-GISEL-NEXT: machine-sink
-; GCN-O2-GISEL-NEXT: peephole-opt
-; GCN-O2-GISEL-NEXT: dead-mi-elimination
-; GCN-O2-GISEL-NEXT: si-fold-operands
-; GCN-O2-GISEL-NEXT: gcn-dpp-combine
-; GCN-O2-GISEL-NEXT: si-load-store-opt
-; GCN-O2-GISEL-NEXT: si-peephole-sdwa
-; GCN-O2-GISEL-NEXT: early-machinelicm
-; GCN-O2-GISEL-NEXT: machine-cse
-; GCN-O2-GISEL-NEXT: si-fold-operands
-; GCN-O2-GISEL-NEXT: dead-mi-elimination
-; GCN-O2-GISEL-NEXT: si-shrink-instructions
-; GCN-O2-GISEL-NEXT: require<reg-usage>
-; GCN-O2-GISEL-NEXT: cgscc
-; GCN-O2-GISEL-NEXT: function
-; GCN-O2-GISEL-NEXT: machine-function
-; GCN-O2-GISEL-NEXT: reg-usage-propagation
-; GCN-O2-GISEL-NEXT: amdgpu-prepare-agpr-alloc
-; GCN-O2-GISEL-NEXT: detect-dead-lanes
-; GCN-O2-GISEL-NEXT: dead-mi-elimination
-; GCN-O2-GISEL-NEXT: init-undef
-; GCN-O2-GISEL-NEXT: process-imp-defs
-; GCN-O2-GISEL-NEXT: unreachable-mbb-elimination
-; GCN-O2-GISEL-NEXT: require<live-vars>
-; GCN-O2-GISEL-NEXT: si-opt-vgpr-liverange
-; GCN-O2-GISEL-NEXT: require<machine-loops>
-; GCN-O2-GISEL-NEXT: phi-node-elimination
-; GCN-O2-GISEL-NEXT: si-lower-control-flow
-; GCN-O2-GISEL-NEXT: two-address-instruction
-; GCN-O2-GISEL-NEXT: register-coalescer
-; GCN-O2-GISEL-NEXT: rename-independent-subregs
-; GCN-O2-GISEL-NEXT: amdgpu-rewrite-partial-reg-uses
-; GCN-O2-GISEL-NEXT: machine-scheduler
-; GCN-O2-GISEL-NEXT: amdgpu-pre-ra-optimizations
-; GCN-O2-GISEL-NEXT: si-wqm
-; GCN-O2-GISEL-NEXT: si-optimize-exec-masking-pre-ra
-; GCN-O2-GISEL-NEXT: si-form-memory-clauses
-; GCN-O2-GISEL-NEXT: amdgpu-pre-ra-long-branch-reg
-; GCN-O2-GISEL-NEXT: greedy<sgpr>
-; GCN-O2-GISEL-NEXT: virt-reg-rewriter<no-clear-vregs>
-; GCN-O2-GISEL-NEXT: stack-slot-coloring
-; GCN-O2-GISEL-NEXT: si-lower-sgpr-spills
-; GCN-O2-GISEL-NEXT: si-pre-allocate-wwm-regs
-; GCN-O2-GISEL-NEXT: greedy<wwm>
-; GCN-O2-GISEL-NEXT: si-lower-wwm-copies
-; GCN-O2-GISEL-NEXT: virt-reg-rewriter<no-clear-vregs>
-; GCN-O2-GISEL-NEXT: amdgpu-reserve-wwm-regs
-; GCN-O2-GISEL-NEXT: greedy<vgpr>
-; GCN-O2-GISEL-NEXT: amdgpu-nsa-reassign
-; GCN-O2-GISEL-NEXT: amdgpu-rewrite-agpr-copy-mfma
-; GCN-O2-GISEL-NEXT: virt-reg-rewriter
-; GCN-O2-GISEL-NEXT: amdgpu-mark-last-scratch-load
-; GCN-O2-GISEL-NEXT: stack-slot-coloring
-; GCN-O2-GISEL-NEXT: machine-cp
-; GCN-O2-GISEL-NEXT: machinelicm
-; GCN-O2-GISEL-NEXT: si-fix-vgpr-copies
-; GCN-O2-GISEL-NEXT: si-optimize-exec-masking
-; GCN-O2-GISEL-NEXT: remove-redundant-debug-values
-; GCN-O2-GISEL-NEXT: fixup-statepoint-caller-saved
-; GCN-O2-GISEL-NEXT: postra-machine-sink
-; GCN-O2-GISEL-NEXT: shrink-wrap
-; GCN-O2-GISEL-NEXT: prolog-epilog
-; GCN-O2-GISEL-NEXT: machine-latecleanup
-; GCN-O2-GISEL-NEXT: branch-folder
-; GCN-O2-GISEL-NEXT: tailduplication
-; GCN-O2-GISEL-NEXT: machine-cp
-; GCN-O2-GISEL-NEXT: post-ra-pseudos
-; GCN-O2-GISEL-NEXT: si-shrink-instructions
-; GCN-O2-GISEL-NEXT: si-post-ra-bundler
-; GCN-O2-GISEL-NEXT: postmisched
-; GCN-O2-GISEL-NEXT: block-placement
-; GCN-O2-GISEL-NEXT: fentry-insert
-; GCN-O2-GISEL-NEXT: xray-instrumentation
-; GCN-O2-GISEL-NEXT: gcn-create-vopd
-; GCN-O2-GISEL-NEXT: si-memory-legalizer
-; GCN-O2-GISEL-NEXT: si-insert-waitcnts
-; GCN-O2-GISEL-NEXT: si-mode-register
-; GCN-O2-GISEL-NEXT: si-insert-hard-clauses
-; GCN-O2-GISEL-NEXT: si-late-branch-lowering
-; GCN-O2-GISEL-NEXT: si-pre-emit-peephole
-; GCN-O2-GISEL-NEXT: post-RA-hazard-rec
-; GCN-O2-GISEL-NEXT: amdgpu-wait-sgpr-hazards
-; GCN-O2-GISEL-NEXT: amdgpu-lower-vgpr-encoding
-; GCN-O2-GISEL-NEXT: amdgpu-insert-delay-alu
-; GCN-O2-GISEL-NEXT: branch-relaxation
-; GCN-O2-GISEL-NEXT: reg-usage-collector
-; GCN-O2-GISEL-NEXT: remove-loads-into-fake-uses
-; GCN-O2-GISEL-NEXT: live-debug-values
-; GCN-O2-GISEL-NEXT: machine-sanmd
-; GCN-O2-GISEL-NEXT: amdgpu-preload-kern-arg-prolog
-; GCN-O2-GISEL-NEXT: stack-frame-layout
-; GCN-O2-GISEL-NEXT: amdgpu-asm-printer
-; GCN-O2-GISEL-NEXT: free-machine-function
-; GCN-O2-GISEL-NEXT: amdgpu-asm-printer-end
-
-; GCN-O3-SDAG: require<MachineModuleAnalysis>
-; GCN-O3-SDAG-NEXT: require<profile-summary>
-; GCN-O3-SDAG-NEXT: require<collector-metadata>
-; GCN-O3-SDAG-NEXT: require<runtime-libcall-info>
-; GCN-O3-SDAG-NEXT: require<libcall-lowering-info>
-; GCN-O3-SDAG-NEXT: function
-; GCN-O3-SDAG-NEXT: objc-arc-contract
-; GCN-O3-SDAG-NEXT: pre-isel-intrinsic-lowering
-; GCN-O3-SDAG-NEXT: function
-; GCN-O3-SDAG-NEXT: expand-ir-insts<O3>
-; GCN-O3-SDAG-NEXT: amdgpu-remove-incompatible-functions
-; GCN-O3-SDAG-NEXT: amdgpu-printf-runtime-binding
-; GCN-O3-SDAG-NEXT: amdgpu-lower-ctor-dtor
-; GCN-O3-SDAG-NEXT: function
-; GCN-O3-SDAG-NEXT: amdgpu-image-intrinsic-opt
-; GCN-O3-SDAG-NEXT: amdgpu-uniform-intrinsic-combine
-; GCN-O3-SDAG-NEXT: expand-variadics
-; GCN-O3-SDAG-NEXT: amdgpu-always-inline
-; GCN-O3-SDAG-NEXT: always-inline
-; GCN-O3-SDAG-NEXT: amdgpu-export-kernel-runtime-handles
-; GCN-O3-SDAG-NEXT: amdgpu-lower-exec-sync
-; GCN-O3-SDAG-NEXT: amdgpu-sw-lower-lds
-; GCN-O3-SDAG-NEXT: amdgpu-lower-module-lds
-; GCN-O3-SDAG-NEXT: function
-; GCN-O3-SDAG-NEXT: amdgpu-atomic-optimizer
-; GCN-O3-SDAG-NEXT: atomic-expand
-; GCN-O3-SDAG-NEXT: amdgpu-promote-alloca
-; GCN-O3-SDAG-NEXT: separate-const-offset-from-gep<>
-; GCN-O3-SDAG-NEXT: slsr
-; GCN-O3-SDAG-NEXT: gvn<>
-; GCN-O3-SDAG-NEXT: nary-reassociate
-; GCN-O3-SDAG-NEXT: early-cse<>
-; GCN-O3-SDAG-NEXT: amdgpu-codegenprepare
-; GCN-O3-SDAG-NEXT: loop-mssa
-; GCN-O3-SDAG-NEXT: licm<allowspeculation>
-; GCN-O3-SDAG-NEXT: verify
-; GCN-O3-SDAG-NEXT: loop
-; GCN-O3-SDAG-NEXT: canon-freeze
-; GCN-O3-SDAG-NEXT: loop-reduce
-; GCN-O3-SDAG-NEXT: unreachableblockelim
-; GCN-O3-SDAG-NEXT: consthoist
-; GCN-O3-SDAG-NEXT: replace-with-veclib
-; GCN-O3-SDAG-NEXT: partially-inline-libcalls
-; GCN-O3-SDAG-NEXT: ee-instrument<post-inline>
-; GCN-O3-SDAG-NEXT: scalarize-masked-mem-intrin
-; GCN-O3-SDAG-NEXT: expand-reductions
-; GCN-O3-SDAG-NEXT: gvn<>
-; GCN-O3-SDAG-NEXT: amdgpu-preload-kernel-arguments
-; GCN-O3-SDAG-NEXT: function
-; GCN-O3-SDAG-NEXT: amdgpu-lower-kernel-arguments
-; GCN-O3-SDAG-NEXT: codegenprepare
-; GCN-O3-SDAG-NEXT: load-store-vectorizer
-; GCN-O3-SDAG-NEXT: amdgpu-lower-buffer-fat-pointers
-; GCN-O3-SDAG-NEXT: amdgpu-lower-intrinsics
-; GCN-O3-SDAG-NEXT: cgscc
-; GCN-O3-SDAG-NEXT: function
-; GCN-O3-SDAG-NEXT: lower-switch
-; GCN-O3-SDAG-NEXT: lower-invoke
-; GCN-O3-SDAG-NEXT: unreachableblockelim
-; GCN-O3-SDAG-NEXT: flatten-cfg
-; GCN-O3-SDAG-NEXT: sink
-; GCN-O3-SDAG-NEXT: amdgpu-late-codegenprepare
-; GCN-O3-SDAG-NEXT: amdgpu-unify-divergent-exit-nodes
-; GCN-O3-SDAG-NEXT: fix-irreducible
-; GCN-O3-SDAG-NEXT: unify-loop-exits
-; GCN-O3-SDAG-NEXT: StructurizeCFGPass
-; GCN-O3-SDAG-NEXT: amdgpu-annotate-uniform
-; GCN-O3-SDAG-NEXT: si-annotate-control-flow
-; GCN-O3-SDAG-NEXT: amdgpu-rewrite-undef-for-phi
-; GCN-O3-SDAG-NEXT: lcssa
-; GCN-O3-SDAG-NEXT: amdgpu-perf-hint
-; GCN-O3-SDAG-NEXT: cgscc
-; GCN-O3-SDAG-NEXT: function
-; GCN-O3-SDAG-NEXT: inline-asm-prepare
-; GCN-O3-SDAG-NEXT: safe-stack
-; GCN-O3-SDAG-NEXT: stack-protector
-; GCN-O3-SDAG-NEXT: verify
-; GCN-O3-SDAG-NEXT: amdgpu-asm-printer-begin
-; GCN-O3-SDAG-NEXT: cgscc
-; GCN-O3-SDAG-NEXT: function
-; GCN-O3-SDAG-NEXT: machine-function
-; GCN-O3-SDAG-NEXT: amdgpu-isel
-; GCN-O3-SDAG-NEXT: si-fix-sgpr-copies
-; GCN-O3-SDAG-NEXT: si-i1-copies
-; GCN-O3-SDAG-NEXT: finalize-isel
-; GCN-O3-SDAG-NEXT: early-tailduplication
-; GCN-O3-SDAG-NEXT: opt-phis
-; GCN-O3-SDAG-NEXT: stack-coloring
-; GCN-O3-SDAG-NEXT: localstackalloc
-; GCN-O3-SDAG-NEXT: dead-mi-elimination
-; GCN-O3-SDAG-NEXT: early-machinelicm
-; GCN-O3-SDAG-NEXT: machine-cse
-; GCN-O3-SDAG-NEXT: machine-sink
-; GCN-O3-SDAG-NEXT: peephole-opt
-; GCN-O3-SDAG-NEXT: dead-mi-elimination
-; GCN-O3-SDAG-NEXT: si-fold-operands
-; GCN-O3-SDAG-NEXT: gcn-dpp-combine
-; GCN-O3-SDAG-NEXT: si-load-store-opt
-; GCN-O3-SDAG-NEXT: si-peephole-sdwa
-; GCN-O3-SDAG-NEXT: early-machinelicm
-; GCN-O3-SDAG-NEXT: machine-cse
-; GCN-O3-SDAG-NEXT: si-fold-operands
-; GCN-O3-SDAG-NEXT: dead-mi-elimination
-; GCN-O3-SDAG-NEXT: si-shrink-instructions
-; GCN-O3-SDAG-NEXT: require<reg-usage>
-; GCN-O3-SDAG-NEXT: cgscc
-; GCN-O3-SDAG-NEXT: function
-; GCN-O3-SDAG-NEXT: machine-function
-; GCN-O3-SDAG-NEXT: reg-usage-propagation
-; GCN-O3-SDAG-NEXT: amdgpu-prepare-agpr-alloc
-; GCN-O3-SDAG-NEXT: detect-dead-lanes
-; GCN-O3-SDAG-NEXT: dead-mi-elimination
-; GCN-O3-SDAG-NEXT: init-undef
-; GCN-O3-SDAG-NEXT: process-imp-defs
-; GCN-O3-SDAG-NEXT: unreachable-mbb-elimination
-; GCN-O3-SDAG-NEXT: require<live-vars>
-; GCN-O3-SDAG-NEXT: si-opt-vgpr-liverange
-; GCN-O3-SDAG-NEXT: require<machine-loops>
-; GCN-O3-SDAG-NEXT: phi-node-elimination
-; GCN-O3-SDAG-NEXT: si-lower-control-flow
-; GCN-O3-SDAG-NEXT: two-address-instruction
-; GCN-O3-SDAG-NEXT: register-coalescer
-; GCN-O3-SDAG-NEXT: rename-independent-subregs
-; GCN-O3-SDAG-NEXT: amdgpu-rewrite-partial-reg-uses
-; GCN-O3-SDAG-NEXT: machine-scheduler
-; GCN-O3-SDAG-NEXT: amdgpu-pre-ra-optimizations
-; GCN-O3-SDAG-NEXT: si-wqm
-; GCN-O3-SDAG-NEXT: si-optimize-exec-masking-pre-ra
-; GCN-O3-SDAG-NEXT: si-form-memory-clauses
-; GCN-O3-SDAG-NEXT: amdgpu-pre-ra-long-branch-reg
-; GCN-O3-SDAG-NEXT: greedy<sgpr>
-; GCN-O3-SDAG-NEXT: virt-reg-rewriter<no-clear-vregs>
-; GCN-O3-SDAG-NEXT: stack-slot-coloring
-; GCN-O3-SDAG-NEXT: si-lower-sgpr-spills
-; GCN-O3-SDAG-NEXT: si-pre-allocate-wwm-regs
-; GCN-O3-SDAG-NEXT: greedy<wwm>
-; GCN-O3-SDAG-NEXT: si-lower-wwm-copies
-; GCN-O3-SDAG-NEXT: virt-reg-rewriter<no-clear-vregs>
-; GCN-O3-SDAG-NEXT: amdgpu-reserve-wwm-regs
-; GCN-O3-SDAG-NEXT: greedy<vgpr>
-; GCN-O3-SDAG-NEXT: amdgpu-nsa-reassign
-; GCN-O3-SDAG-NEXT: amdgpu-rewrite-agpr-copy-mfma
-; GCN-O3-SDAG-NEXT: virt-reg-rewriter
-; GCN-O3-SDAG-NEXT: amdgpu-mark-last-scratch-load
-; GCN-O3-SDAG-NEXT: stack-slot-coloring
-; GCN-O3-SDAG-NEXT: machine-cp
-; GCN-O3-SDAG-NEXT: machinelicm
-; GCN-O3-SDAG-NEXT: si-fix-vgpr-copies
-; GCN-O3-SDAG-NEXT: si-optimize-exec-masking
-; GCN-O3-SDAG-NEXT: remove-redundant-debug-values
-; GCN-O3-SDAG-NEXT: fixup-statepoint-caller-saved
-; GCN-O3-SDAG-NEXT: postra-machine-sink
-; GCN-O3-SDAG-NEXT: shrink-wrap
-; GCN-O3-SDAG-NEXT: prolog-epilog
-; GCN-O3-SDAG-NEXT: machine-latecleanup
-; GCN-O3-SDAG-NEXT: branch-folder
-; GCN-O3-SDAG-NEXT: tailduplication
-; GCN-O3-SDAG-NEXT: machine-cp
-; GCN-O3-SDAG-NEXT: post-ra-pseudos
-; GCN-O3-SDAG-NEXT: si-shrink-instructions
-; GCN-O3-SDAG-NEXT: si-post-ra-bundler
-; GCN-O3-SDAG-NEXT: postmisched
-; GCN-O3-SDAG-NEXT: block-placement
-; GCN-O3-SDAG-NEXT: fentry-insert
-; GCN-O3-SDAG-NEXT: xray-instrumentation
-; GCN-O3-SDAG-NEXT: gcn-create-vopd
-; GCN-O3-SDAG-NEXT: si-memory-legalizer
-; GCN-O3-SDAG-NEXT: si-insert-waitcnts
-; GCN-O3-SDAG-NEXT: si-mode-register
-; GCN-O3-SDAG-NEXT: si-insert-hard-clauses
-; GCN-O3-SDAG-NEXT: si-late-branch-lowering
-; GCN-O3-SDAG-NEXT: si-pre-emit-peephole
-; GCN-O3-SDAG-NEXT: post-RA-hazard-rec
-; GCN-O3-SDAG-NEXT: amdgpu-wait-sgpr-hazards
-; GCN-O3-SDAG-NEXT: amdgpu-lower-vgpr-encoding
-; GCN-O3-SDAG-NEXT: amdgpu-insert-delay-alu
-; GCN-O3-SDAG-NEXT: branch-relaxation
-; GCN-O3-SDAG-NEXT: reg-usage-collector
-; GCN-O3-SDAG-NEXT: remove-loads-into-fake-uses
-; GCN-O3-SDAG-NEXT: live-debug-values
-; GCN-O3-SDAG-NEXT: machine-sanmd
-; GCN-O3-SDAG-NEXT: amdgpu-preload-kern-arg-prolog
-; GCN-O3-SDAG-NEXT: stack-frame-layout
-; GCN-O3-SDAG-NEXT: amdgpu-asm-printer
-; GCN-O3-SDAG-NEXT: free-machine-function
-; GCN-O3-SDAG-NEXT: amdgpu-asm-printer-end
-
-; GCN-O3-GISEL: require<MachineModuleAnalysis>
-; GCN-O3-GISEL-NEXT: require<profile-summary>
-; GCN-O3-GISEL-NEXT: require<collector-metadata>
-; GCN-O3-GISEL-NEXT: require<runtime-libcall-info>
-; GCN-O3-GISEL-NEXT: require<libcall-lowering-info>
-; GCN-O3-GISEL-NEXT: function
-; GCN-O3-GISEL-NEXT: objc-arc-contract
-; GCN-O3-GISEL-NEXT: pre-isel-intrinsic-lowering
-; GCN-O3-GISEL-NEXT: function
-; GCN-O3-GISEL-NEXT: expand-ir-insts<O3>
-; GCN-O3-GISEL-NEXT: amdgpu-remove-incompatible-functions
-; GCN-O3-GISEL-NEXT: amdgpu-printf-runtime-binding
-; GCN-O3-GISEL-NEXT: amdgpu-lower-ctor-dtor
-; GCN-O3-GISEL-NEXT: function
-; GCN-O3-GISEL-NEXT: amdgpu-image-intrinsic-opt
-; GCN-O3-GISEL-NEXT: amdgpu-uniform-intrinsic-combine
-; GCN-O3-GISEL-NEXT: expand-variadics
-; GCN-O3-GISEL-NEXT: amdgpu-always-inline
-; GCN-O3-GISEL-NEXT: always-inline
-; GCN-O3-GISEL-NEXT: amdgpu-export-kernel-runtime-handles
-; GCN-O3-GISEL-NEXT: amdgpu-lower-exec-sync
-; GCN-O3-GISEL-NEXT: amdgpu-sw-lower-lds
-; GCN-O3-GISEL-NEXT: amdgpu-lower-module-lds
-; GCN-O3-GISEL-NEXT: function
-; GCN-O3-GISEL-NEXT: amdgpu-atomic-optimizer
-; GCN-O3-GISEL-NEXT: atomic-expand
-; GCN-O3-GISEL-NEXT: amdgpu-promote-alloca
-; GCN-O3-GISEL-NEXT: separate-const-offset-from-gep<>
-; GCN-O3-GISEL-NEXT: slsr
-; GCN-O3-GISEL-NEXT: gvn<>
-; GCN-O3-GISEL-NEXT: nary-reassociate
-; GCN-O3-GISEL-NEXT: early-cse<>
-; GCN-O3-GISEL-NEXT: amdgpu-codegenprepare
-; GCN-O3-GISEL-NEXT: loop-mssa
-; GCN-O3-GISEL-NEXT: licm<allowspeculation>
-; GCN-O3-GISEL-NEXT: verify
-; GCN-O3-GISEL-NEXT: loop
-; GCN-O3-GISEL-NEXT: canon-freeze
-; GCN-O3-GISEL-NEXT: loop-reduce
-; GCN-O3-GISEL-NEXT: unreachableblockelim
-; GCN-O3-GISEL-NEXT: consthoist
-; GCN-O3-GISEL-NEXT: replace-with-veclib
-; GCN-O3-GISEL-NEXT: partially-inline-libcalls
-; GCN-O3-GISEL-NEXT: ee-instrument<post-inline>
-; GCN-O3-GISEL-NEXT: scalarize-masked-mem-intrin
-; GCN-O3-GISEL-NEXT: expand-reductions
-; GCN-O3-GISEL-NEXT: gvn<>
-; GCN-O3-GISEL-NEXT: amdgpu-preload-kernel-arguments
-; GCN-O3-GISEL-NEXT: function
-; GCN-O3-GISEL-NEXT: amdgpu-lower-kernel-arguments
-; GCN-O3-GISEL-NEXT: codegenprepare
-; GCN-O3-GISEL-NEXT: load-store-vectorizer
-; GCN-O3-GISEL-NEXT: amdgpu-lower-buffer-fat-pointers
-; GCN-O3-GISEL-NEXT: amdgpu-lower-intrinsics
-; GCN-O3-GISEL-NEXT: cgscc
-; GCN-O3-GISEL-NEXT: function
-; GCN-O3-GISEL-NEXT: lower-switch
-; GCN-O3-GISEL-NEXT: lower-invoke
-; GCN-O3-GISEL-NEXT: unreachableblockelim
-; GCN-O3-GISEL-NEXT: flatten-cfg
-; GCN-O3-GISEL-NEXT: sink
-; GCN-O3-GISEL-NEXT: amdgpu-late-codegenprepare
-; GCN-O3-GISEL-NEXT: amdgpu-unify-divergent-exit-nodes
-; GCN-O3-GISEL-NEXT: fix-irreducible
-; GCN-O3-GISEL-NEXT: unify-loop-exits
-; GCN-O3-GISEL-NEXT: StructurizeCFGPass
-; GCN-O3-GISEL-NEXT: amdgpu-annotate-uniform
-; GCN-O3-GISEL-NEXT: si-annotate-control-flow
-; GCN-O3-GISEL-NEXT: amdgpu-rewrite-undef-for-phi
-; GCN-O3-GISEL-NEXT: amdgpu-perf-hint
-; GCN-O3-GISEL-NEXT: cgscc
-; GCN-O3-GISEL-NEXT: function
-; GCN-O3-GISEL-NEXT: inline-asm-prepare
-; GCN-O3-GISEL-NEXT: safe-stack
-; GCN-O3-GISEL-NEXT: stack-protector
-; GCN-O3-GISEL-NEXT: verify
-; GCN-O3-GISEL-NEXT: amdgpu-asm-printer-begin
-; GCN-O3-GISEL-NEXT: cgscc
-; GCN-O3-GISEL-NEXT: function
-; GCN-O3-GISEL-NEXT: machine-function
-; GCN-O3-GISEL-NEXT: ir-translator
-; GCN-O3-GISEL-NEXT: amdgpu-prelegalizer-combiner
-; GCN-O3-GISEL-NEXT: localizer
-; GCN-O3-GISEL-NEXT: legalizer
-; GCN-O3-GISEL-NEXT: amdgpu-postlegalizer-combiner
-; GCN-O3-GISEL-NEXT: amdgpu-global-isel-divergence-lowering
-; GCN-O3-GISEL-NEXT: amdgpu-reg-bank-select
-; GCN-O3-GISEL-NEXT: amdgpu-reg-bank-legalize
-; GCN-O3-GISEL-NEXT: amdgpu-regbank-combiner
-; GCN-O3-GISEL-NEXT: instruction-select
-; GCN-O3-GISEL-NEXT: reset-machine-function
-; GCN-O3-GISEL-NEXT: finalize-isel
-; GCN-O3-GISEL-NEXT: early-tailduplication
-; GCN-O3-GISEL-NEXT: opt-phis
-; GCN-O3-GISEL-NEXT: stack-coloring
-; GCN-O3-GISEL-NEXT: localstackalloc
-; GCN-O3-GISEL-NEXT: dead-mi-elimination
-; GCN-O3-GISEL-NEXT: early-machinelicm
-; GCN-O3-GISEL-NEXT: machine-cse
-; GCN-O3-GISEL-NEXT: machine-sink
-; GCN-O3-GISEL-NEXT: peephole-opt
-; GCN-O3-GISEL-NEXT: dead-mi-elimination
-; GCN-O3-GISEL-NEXT: si-fold-operands
-; GCN-O3-GISEL-NEXT: gcn-dpp-combine
-; GCN-O3-GISEL-NEXT: si-load-store-opt
-; GCN-O3-GISEL-NEXT: si-peephole-sdwa
-; GCN-O3-GISEL-NEXT: early-machinelicm
-; GCN-O3-GISEL-NEXT: machine-cse
-; GCN-O3-GISEL-NEXT: si-fold-operands
-; GCN-O3-GISEL-NEXT: dead-mi-elimination
-; GCN-O3-GISEL-NEXT: si-shrink-instructions
-; GCN-O3-GISEL-NEXT: require<reg-usage>
-; GCN-O3-GISEL-NEXT: cgscc
-; GCN-O3-GISEL-NEXT: function
-; GCN-O3-GISEL-NEXT: machine-function
-; GCN-O3-GISEL-NEXT: reg-usage-propagation
-; GCN-O3-GISEL-NEXT: amdgpu-prepare-agpr-alloc
-; GCN-O3-GISEL-NEXT: detect-dead-lanes
-; GCN-O3-GISEL-NEXT: dead-mi-elimination
-; GCN-O3-GISEL-NEXT: init-undef
-; GCN-O3-GISEL-NEXT: process-imp-defs
-; GCN-O3-GISEL-NEXT: unreachable-mbb-elimination
-; GCN-O3-GISEL-NEXT: require<live-vars>
-; GCN-O3-GISEL-NEXT: si-opt-vgpr-liverange
-; GCN-O3-GISEL-NEXT: require<machine-loops>
-; GCN-O3-GISEL-NEXT: phi-node-elimination
-; GCN-O3-GISEL-NEXT: si-lower-control-flow
-; GCN-O3-GISEL-NEXT: two-address-instruction
-; GCN-O3-GISEL-NEXT: register-coalescer
-; GCN-O3-GISEL-NEXT: rename-independent-subregs
-; GCN-O3-GISEL-NEXT: amdgpu-rewrite-partial-reg-uses
-; GCN-O3-GISEL-NEXT: machine-scheduler
-; GCN-O3-GISEL-NEXT: amdgpu-pre-ra-optimizations
-; GCN-O3-GISEL-NEXT: si-wqm
-; GCN-O3-GISEL-NEXT: si-optimize-exec-masking-pre-ra
-; GCN-O3-GISEL-NEXT: si-form-memory-clauses
-; GCN-O3-GISEL-NEXT: amdgpu-pre-ra-long-branch-reg
-; GCN-O3-GISEL-NEXT: greedy<sgpr>
-; GCN-O3-GISEL-NEXT: virt-reg-rewriter<no-clear-vregs>
-; GCN-O3-GISEL-NEXT: stack-slot-coloring
-; GCN-O3-GISEL-NEXT: si-lower-sgpr-spills
-; GCN-O3-GISEL-NEXT: si-pre-allocate-wwm-regs
-; GCN-O3-GISEL-NEXT: greedy<wwm>
-; GCN-O3-GISEL-NEXT: si-lower-wwm-copies
-; GCN-O3-GISEL-NEXT: virt-reg-rewriter<no-clear-vregs>
-; GCN-O3-GISEL-NEXT: amdgpu-reserve-wwm-regs
-; GCN-O3-GISEL-NEXT: greedy<vgpr>
-; GCN-O3-GISEL-NEXT: amdgpu-nsa-reassign
-; GCN-O3-GISEL-NEXT: amdgpu-rewrite-agpr-copy-mfma
-; GCN-O3-GISEL-NEXT: virt-reg-rewriter
-; GCN-O3-GISEL-NEXT: amdgpu-mark-last-scratch-load
-; GCN-O3-GISEL-NEXT: stack-slot-coloring
-; GCN-O3-GISEL-NEXT: machine-cp
-; GCN-O3-GISEL-NEXT: machinelicm
-; GCN-O3-GISEL-NEXT: si-fix-vgpr-copies
-; GCN-O3-GISEL-NEXT: si-optimize-exec-masking
-; GCN-O3-GISEL-NEXT: remove-redundant-debug-values
-; GCN-O3-GISEL-NEXT: fixup-statepoint-caller-saved
-; GCN-O3-GISEL-NEXT: postra-machine-sink
-; GCN-O3-GISEL-NEXT: shrink-wrap
-; GCN-O3-GISEL-NEXT: prolog-epilog
-; GCN-O3-GISEL-NEXT: machine-latecleanup
-; GCN-O3-GISEL-NEXT: branch-folder
-; GCN-O3-GISEL-NEXT: tailduplication
-; GCN-O3-GISEL-NEXT: machine-cp
-; GCN-O3-GISEL-NEXT: post-ra-pseudos
-; GCN-O3-GISEL-NEXT: si-shrink-instructions
-; GCN-O3-GISEL-NEXT: si-post-ra-bundler
-; GCN-O3-GISEL-NEXT: postmisched
-; GCN-O3-GISEL-NEXT: block-placement
-; GCN-O3-GISEL-NEXT: fentry-insert
-; GCN-O3-GISEL-NEXT: xray-instrumentation
-; GCN-O3-GISEL-NEXT: gcn-create-vopd
-; GCN-O3-GISEL-NEXT: si-memory-legalizer
-; GCN-O3-GISEL-NEXT: si-insert-waitcnts
-; GCN-O3-GISEL-NEXT: si-mode-register
-; GCN-O3-GISEL-NEXT: si-insert-hard-clauses
-; GCN-O3-GISEL-NEXT: si-late-branch-lowering
-; GCN-O3-GISEL-NEXT: si-pre-emit-peephole
-; GCN-O3-GISEL-NEXT: post-RA-hazard-rec
-; GCN-O3-GISEL-NEXT: amdgpu-wait-sgpr-hazards
-; GCN-O3-GISEL-NEXT: amdgpu-lower-vgpr-encoding
-; GCN-O3-GISEL-NEXT: amdgpu-insert-delay-alu
-; GCN-O3-GISEL-NEXT: branch-relaxation
-; GCN-O3-GISEL-NEXT: reg-usage-collector
-; GCN-O3-GISEL-NEXT: remove-loads-into-fake-uses
-; GCN-O3-GISEL-NEXT: live-debug-values
-; GCN-O3-GISEL-NEXT: machine-sanmd
-; GCN-O3-GISEL-NEXT: amdgpu-preload-kern-arg-prolog
-; GCN-O3-GISEL-NEXT: stack-frame-layout
-; GCN-O3-GISEL-NEXT: amdgpu-asm-printer
-; GCN-O3-GISEL-NEXT: free-machine-function
-; GCN-O3-GISEL-NEXT: amdgpu-asm-printer-end
+; GCN-O3: require<MachineModuleAnalysis>
+; GCN-O3-NEXT: require<profile-summary>
+; GCN-O3-NEXT: require<collector-metadata>
+; GCN-O3-NEXT: require<runtime-libcall-info>
+; GCN-O3-NEXT: require<libcall-lowering-info>
+; GCN-O3-NEXT: function
+; GCN-O3-NEXT: objc-arc-contract
+; GCN-O3-NEXT: pre-isel-intrinsic-lowering
+; GCN-O3-NEXT: function
+; GCN-O3-NEXT: expand-ir-insts<O3>
+; GCN-O3-NEXT: amdgpu-remove-incompatible-functions
+; GCN-O3-NEXT: amdgpu-printf-runtime-binding
+; GCN-O3-NEXT: amdgpu-lower-ctor-dtor
+; GCN-O3-NEXT: function
+; GCN-O3-NEXT: amdgpu-image-intrinsic-opt
+; GCN-O3-NEXT: amdgpu-uniform-intrinsic-combine
+; GCN-O3-NEXT: expand-variadics
+; GCN-O3-NEXT: amdgpu-always-inline
+; GCN-O3-NEXT: always-inline
+; GCN-O3-NEXT: amdgpu-export-kernel-runtime-handles
+; GCN-O3-NEXT: amdgpu-lower-exec-sync
+; GCN-O3-NEXT: amdgpu-sw-lower-lds
+; GCN-O3-NEXT: amdgpu-lower-module-lds
+; GCN-O3-NEXT: function
+; GCN-O3-NEXT: amdgpu-atomic-optimizer
+; GCN-O3-NEXT: atomic-expand
+; GCN-O3-NEXT: amdgpu-promote-alloca
+; GCN-O3-NEXT: separate-const-offset-from-gep<>
+; GCN-O3-NEXT: slsr
+; GCN-O3-NEXT: gvn<>
+; GCN-O3-NEXT: nary-reassociate
+; GCN-O3-NEXT: early-cse<>
+; GCN-O3-NEXT: amdgpu-codegenprepare
+; GCN-O3-NEXT: loop-mssa
+; GCN-O3-NEXT: licm<allowspeculation>
+; GCN-O3-NEXT: verify
+; GCN-O3-NEXT: loop
+; GCN-O3-NEXT: canon-freeze
+; GCN-O3-NEXT: loop-reduce
+; GCN-O3-NEXT: unreachableblockelim
+; GCN-O3-NEXT: consthoist
+; GCN-O3-NEXT: replace-with-veclib
+; GCN-O3-NEXT: partially-inline-libcalls
+; GCN-O3-NEXT: ee-instrument<post-inline>
+; GCN-O3-NEXT: scalarize-masked-mem-intrin
+; GCN-O3-NEXT: expand-reductions
+; GCN-O3-NEXT: gvn<>
+; GCN-O3-NEXT: amdgpu-preload-kernel-arguments
+; GCN-O3-NEXT: function
+; GCN-O3-NEXT: amdgpu-lower-kernel-arguments
+; GCN-O3-NEXT: codegenprepare
+; GCN-O3-NEXT: load-store-vectorizer
+; GCN-O3-NEXT: amdgpu-lower-buffer-fat-pointers
+; GCN-O3-NEXT: amdgpu-lower-intrinsics
+; GCN-O3-NEXT: cgscc
+; GCN-O3-NEXT: function
+; GCN-O3-NEXT: lower-switch
+; GCN-O3-NEXT: lower-invoke
+; GCN-O3-NEXT: unreachableblockelim
+; GCN-O3-NEXT: flatten-cfg
+; GCN-O3-NEXT: sink
+; GCN-O3-NEXT: amdgpu-late-codegenprepare
+; GCN-O3-NEXT: amdgpu-unify-divergent-exit-nodes
+; GCN-O3-NEXT: fix-irreducible
+; GCN-O3-NEXT: unify-loop-exits
+; GCN-O3-NEXT: StructurizeCFGPass
+; GCN-O3-NEXT: amdgpu-annotate-uniform
+; GCN-O3-NEXT: si-annotate-control-flow
+; GCN-O3-NEXT: amdgpu-rewrite-undef-for-phi
+; GCN-O3-NEXT: lcssa
+; GCN-O3-NEXT: amdgpu-perf-hint
+; GCN-O3-NEXT: cgscc
+; GCN-O3-NEXT: function
+; GCN-O3-NEXT: inline-asm-prepare
+; GCN-O3-NEXT: safe-stack
+; GCN-O3-NEXT: stack-protector
+; GCN-O3-NEXT: verify
+; GCN-O3-NEXT: amdgpu-asm-printer-begin
+; GCN-O3-NEXT: cgscc
+; GCN-O3-NEXT: function
+; GCN-O3-NEXT: machine-function
+; GCN-O3-NEXT: amdgpu-isel
+; GCN-O3-NEXT: si-fix-sgpr-copies
+; GCN-O3-NEXT: si-i1-copies
+; GCN-O3-NEXT: finalize-isel
+; GCN-O3-NEXT: early-tailduplication
+; GCN-O3-NEXT: opt-phis
+; GCN-O3-NEXT: stack-coloring
+; GCN-O3-NEXT: localstackalloc
+; GCN-O3-NEXT: dead-mi-elimination
+; GCN-O3-NEXT: early-machinelicm
+; GCN-O3-NEXT: machine-cse
+; GCN-O3-NEXT: machine-sink
+; GCN-O3-NEXT: peephole-opt
+; GCN-O3-NEXT: dead-mi-elimination
+; GCN-O3-NEXT: si-fold-operands
+; GCN-O3-NEXT: gcn-dpp-combine
+; GCN-O3-NEXT: si-load-store-opt
+; GCN-O3-NEXT: si-peephole-sdwa
+; GCN-O3-NEXT: early-machinelicm
+; GCN-O3-NEXT: machine-cse
+; GCN-O3-NEXT: si-fold-operands
+; GCN-O3-NEXT: dead-mi-elimination
+; GCN-O3-NEXT: si-shrink-instructions
+; GCN-O3-NEXT: require<reg-usage>
+; GCN-O3-NEXT: cgscc
+; GCN-O3-NEXT: function
+; GCN-O3-NEXT: machine-function
+; GCN-O3-NEXT: reg-usage-propagation
+; GCN-O3-NEXT: amdgpu-prepare-agpr-alloc
+; GCN-O3-NEXT: detect-dead-lanes
+; GCN-O3-NEXT: dead-mi-elimination
+; GCN-O3-NEXT: init-undef
+; GCN-O3-NEXT: process-imp-defs
+; GCN-O3-NEXT: unreachable-mbb-elimination
+; GCN-O3-NEXT: require<live-vars>
+; GCN-O3-NEXT: si-opt-vgpr-liverange
+; GCN-O3-NEXT: require<machine-loops>
+; GCN-O3-NEXT: phi-node-elimination
+; GCN-O3-NEXT: si-lower-control-flow
+; GCN-O3-NEXT: two-address-instruction
+; GCN-O3-NEXT: register-coalescer
+; GCN-O3-NEXT: rename-independent-subregs
+; GCN-O3-NEXT: amdgpu-rewrite-partial-reg-uses
+; GCN-O3-NEXT: machine-scheduler
+; GCN-O3-NEXT: amdgpu-pre-ra-optimizations
+; GCN-O3-NEXT: si-wqm
+; GCN-O3-NEXT: si-optimize-exec-masking-pre-ra
+; GCN-O3-NEXT: si-form-memory-clauses
+; GCN-O3-NEXT: amdgpu-pre-ra-long-branch-reg
+; GCN-O3-NEXT: greedy<sgpr>
+; GCN-O3-NEXT: virt-reg-rewriter<no-clear-vregs>
+; GCN-O3-NEXT: stack-slot-coloring
+; GCN-O3-NEXT: si-lower-sgpr-spills
+; GCN-O3-NEXT: si-pre-allocate-wwm-regs
+; GCN-O3-NEXT: greedy<wwm>
+; GCN-O3-NEXT: si-lower-wwm-copies
+; GCN-O3-NEXT: virt-reg-rewriter<no-clear-vregs>
+; GCN-O3-NEXT: amdgpu-reserve-wwm-regs
+; GCN-O3-NEXT: greedy<vgpr>
+; GCN-O3-NEXT: amdgpu-nsa-reassign
+; GCN-O3-NEXT: amdgpu-rewrite-agpr-copy-mfma
+; GCN-O3-NEXT: virt-reg-rewriter
+; GCN-O3-NEXT: amdgpu-mark-last-scratch-load
+; GCN-O3-NEXT: stack-slot-coloring
+; GCN-O3-NEXT: machine-cp
+; GCN-O3-NEXT: machinelicm
+; GCN-O3-NEXT: si-fix-vgpr-copies
+; GCN-O3-NEXT: si-optimize-exec-masking
+; GCN-O3-NEXT: remove-redundant-debug-values
+; GCN-O3-NEXT: fixup-statepoint-caller-saved
+; GCN-O3-NEXT: postra-machine-sink
+; GCN-O3-NEXT: shrink-wrap
+; GCN-O3-NEXT: prolog-epilog
+; GCN-O3-NEXT: machine-latecleanup
+; GCN-O3-NEXT: branch-folder
+; GCN-O3-NEXT: tailduplication
+; GCN-O3-NEXT: machine-cp
+; GCN-O3-NEXT: post-ra-pseudos
+; GCN-O3-NEXT: si-shrink-instructions
+; GCN-O3-NEXT: si-post-ra-bundler
+; GCN-O3-NEXT: postmisched
+; GCN-O3-NEXT: block-placement
+; GCN-O3-NEXT: fentry-insert
+; GCN-O3-NEXT: xray-instrumentation
+; GCN-O3-NEXT: gcn-create-vopd
+; GCN-O3-NEXT: si-memory-legalizer
+; GCN-O3-NEXT: si-post-ra-16bit-mov-folding
+; GCN-O3-NEXT: si-insert-waitcnts
+; GCN-O3-NEXT: si-mode-register
+; GCN-O3-NEXT: si-insert-hard-clauses
+; GCN-O3-NEXT: si-late-branch-lowering
+; GCN-O3-NEXT: si-pre-emit-peephole
+; GCN-O3-NEXT: post-RA-hazard-rec
+; GCN-O3-NEXT: amdgpu-wait-sgpr-hazards
+; GCN-O3-NEXT: amdgpu-lower-vgpr-encoding
+; GCN-O3-NEXT: amdgpu-insert-delay-alu
+; GCN-O3-NEXT: branch-relaxation
+; GCN-O3-NEXT: reg-usage-collector
+; GCN-O3-NEXT: remove-loads-into-fake-uses
+; GCN-O3-NEXT: live-debug-values
+; GCN-O3-NEXT: machine-sanmd
+; GCN-O3-NEXT: amdgpu-preload-kern-arg-prolog
+; GCN-O3-NEXT: stack-frame-layout
+; GCN-O3-NEXT: amdgpu-asm-printer
+; GCN-O3-NEXT: free-machine-function
+; GCN-O3-NEXT: amdgpu-asm-printer-end
define void @empty() {
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index bb7ed3b58f8af..41002382b042f 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -442,6 +442,7 @@
; GCN-O1-NEXT: Insert XRay ops
; GCN-O1-NEXT: GCN Create VOPD Instructions
; GCN-O1-NEXT: SI Memory Legalizer
+; GCN-O1-NEXT: SI post-RA 16bit Mov Folding
; GCN-O1-NEXT: MachineDominator Tree Construction
; GCN-O1-NEXT: Machine Natural Loop Construction
; GCN-O1-NEXT: MachinePostDominator Tree Construction
@@ -771,6 +772,7 @@
; GCN-O1-OPTS-NEXT: Insert XRay ops
; GCN-O1-OPTS-NEXT: GCN Create VOPD Instructions
; GCN-O1-OPTS-NEXT: SI Memory Legalizer
+; GCN-O1-OPTS-NEXT: SI post-RA 16bit Mov Folding
; GCN-O1-OPTS-NEXT: MachineDominator Tree Construction
; GCN-O1-OPTS-NEXT: Machine Natural Loop Construction
; GCN-O1-OPTS-NEXT: MachinePostDominator Tree Construction
@@ -1105,6 +1107,7 @@
; GCN-O2-NEXT: Insert XRay ops
; GCN-O2-NEXT: GCN Create VOPD Instructions
; GCN-O2-NEXT: SI Memory Legalizer
+; GCN-O2-NEXT: SI post-RA 16bit Mov Folding
; GCN-O2-NEXT: MachineDominator Tree Construction
; GCN-O2-NEXT: Machine Natural Loop Construction
; GCN-O2-NEXT: MachinePostDominator Tree Construction
@@ -1454,6 +1457,7 @@
; GCN-O3-NEXT: Insert XRay ops
; GCN-O3-NEXT: GCN Create VOPD Instructions
; GCN-O3-NEXT: SI Memory Legalizer
+; GCN-O3-NEXT: SI post-RA 16bit Mov Folding
; GCN-O3-NEXT: MachineDominator Tree Construction
; GCN-O3-NEXT: Machine Natural Loop Construction
; GCN-O3-NEXT: MachinePostDominator Tree Construction
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
index 87a41bf3e3cc6..03b7c6453eb1f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.gather4.a16.dim.ll
@@ -101,8 +101,7 @@ define amdgpu_ps <4 x float> @gather4_cube(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -169,8 +168,7 @@ define amdgpu_ps <4 x float> @gather4_2darray(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4 v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -303,8 +301,7 @@ define amdgpu_ps <4 x float> @gather4_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inr
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4_cl v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -573,8 +570,7 @@ define amdgpu_ps <4 x float> @gather4_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> i
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v2, v1, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_gather4_b_cl v[0:3], v[2:4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
@@ -703,8 +699,7 @@ define amdgpu_ps <4 x float> @gather4_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-LABEL: gather4_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: image_gather4_l v[0:3], v[2:3], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -902,5 +897,3 @@ declare <4 x float> @llvm.amdgcn.image.gather4.c.lz.2d.v4f32.f32(i32, float, hal
attributes #0 = { nounwind }
attributes #1 = { nounwind readonly }
attributes #2 = { nounwind readnone }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX12: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
index 2f288d6fb95a6..073c1bc510901 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.msaa.load.ll
@@ -305,8 +305,7 @@ define amdgpu_ps <4 x float> @load_2dmsaa_a16(<8 x i32> inreg %rsrc, i16 %s, i16
; GFX11-TRUE16-LABEL: load_2dmsaa_a16:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[0:7] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -339,10 +338,8 @@ main_body:
define amdgpu_ps <4 x float> @load_2darraymsaa_a16(<8 x i32> inreg %rsrc, i16 %s, i16 %t, i16 %slice, i16 %fragid) {
; GFX11-TRUE16-LABEL: load_2darraymsaa_a16:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v3, v2, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[3:4], s[0:7] dmask:0x4 dim:SQ_RSRC_IMG_2D_MSAA_ARRAY unorm a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
index e3c84dcfcc027..4f4298f8d02c1 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.a16.dim.ll
@@ -142,8 +142,7 @@ define amdgpu_ps <4 x float> @sample_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg %
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -210,8 +209,7 @@ define amdgpu_ps <4 x float> @sample_cube(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_CUBE a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -344,8 +342,7 @@ define amdgpu_ps <4 x float> @sample_2darray(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -585,8 +582,7 @@ define amdgpu_ps <4 x float> @sample_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> inre
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample_cl v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -1069,8 +1065,7 @@ define amdgpu_ps <4 x float> @sample_b_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16-NEXT: s_mov_b32 s12, exec_lo
; GFX11-TRUE16-NEXT: s_wqm_b32 exec_lo, exec_lo
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v3, v2, v1, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
; GFX11-TRUE16-NEXT: s_and_b32 exec_lo, exec_lo, s12
; GFX11-TRUE16-NEXT: image_sample_b_cl v[0:3], v[2:4], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
@@ -1297,12 +1292,9 @@ define amdgpu_ps <4 x float> @sample_d_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
;
; GFX11-TRUE16-LABEL: sample_d_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v5, v4, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v3, v2, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], v[4:6], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -1367,8 +1359,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: sample_d_3d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
; GFX11-TRUE16-NEXT: image_sample_d_g16 v[0:3], [v0, v2, v3, v5, v[8:9]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_3D a16
@@ -1387,8 +1378,7 @@ define amdgpu_ps <4 x float> @sample_d_3d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX12PLUS-TRUE16-LABEL: sample_d_3d:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
@@ -1573,12 +1563,9 @@ define amdgpu_ps <4 x float> @sample_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32> in
; GFX11-TRUE16-LABEL: sample_d_cl_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v7.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.l, v4.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v6.h, v5.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v5.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v6, v5, v4, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v5, v3, v2, 0x5040100
+; GFX11-TRUE16-NEXT: v_perm_b32 v4, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: image_sample_d_cl_g16 v[0:3], v[4:7], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -1705,8 +1692,7 @@ define amdgpu_ps <4 x float> @sample_c_d_cl_2d(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12PLUS-TRUE16-LABEL: sample_c_d_cl_2d:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v5.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v6.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v7, v6, v5, 0x5040100
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v3.h, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_cl_g16 v[0:3], [v0, v1, v3, v[7:8]], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
@@ -1793,8 +1779,7 @@ define amdgpu_ps <4 x float> @sample_l_2d(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: sample_l_2d:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: image_sample_l v[0:3], v[2:3], s[0:7], s[8:11] dmask:0xf dim:SQ_RSRC_IMG_2D a16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: ; return to shader part epilog
@@ -2105,8 +2090,7 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v4, v[8:9]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
@@ -2125,10 +2109,8 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v7, v5, v4, 0x5040100
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[7:9]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
@@ -2176,8 +2158,7 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX11-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX11-TRUE16: ; %bb.0: ; %main_body
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v4.h, v5.l
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX11-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v4, v[8:9]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
@@ -2196,10 +2177,8 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v9.l, v8.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.l, v6.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v8.h, v7.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.l, v4.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v7.h, v5.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v8, v7, v6, 0x5040100
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v7, v5, v4, 0x5040100
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[7:9]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY a16
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
index 8bd42ece040d2..1b7b4c2d33a6d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.encode.ll
@@ -512,10 +512,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12-TRUE16: ; %bb.0: ; %main_body
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf]
+; GFX12-TRUE16-NEXT: v_perm_b32 v5, v9, v4, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd6,0x09,0x09,0xfe,0x03,0x00,0x01,0x04,0x05]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x0f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -531,10 +530,9 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX13-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX13-TRUE16: ; %bb.0: ; %main_body
; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
+; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0xae,0xbf]
+; GFX13-TRUE16-NEXT: v_perm_b32 v5, v9, v4, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd7,0x09,0x09,0xfe,0x03,0x00,0x01,0x04,0x05]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0x2e,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX13-TRUE16-NEXT: ; return to shader part epilog
@@ -583,10 +581,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12-TRUE16: ; %bb.0: ; %main_body
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0x87,0xbf]
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0x87,0xbf]
+; GFX12-TRUE16-NEXT: v_perm_b32 v5, v9, v4, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd6,0x09,0x09,0xfe,0x03,0x00,0x01,0x04,0x05]
; GFX12-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x00,0x8f,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX12-TRUE16-NEXT: ; return to shader part epilog
@@ -602,10 +599,9 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX13-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX13-TRUE16: ; %bb.0: ; %main_body
; GFX13-TRUE16-NEXT: v_mov_b32_e32 v9, v5 ; encoding: [0x05,0x03,0x12,0x7e]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l ; encoding: [0x04,0x39,0x0a,0x7e]
; GFX13-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l ; encoding: [0x03,0x39,0x04,0x7f]
-; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) ; encoding: [0x03,0x00,0xae,0xbf]
-; GFX13-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l ; encoding: [0x09,0x39,0x0a,0x7f]
+; GFX13-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) ; encoding: [0x02,0x00,0xae,0xbf]
+; GFX13-TRUE16-NEXT: v_perm_b32 v5, v9, v4, 0x5040100 ; encoding: [0x05,0x00,0x44,0xd7,0x09,0x09,0xfe,0x03,0x00,0x01,0x04,0x05]
; GFX13-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY ; encoding: [0x05,0x80,0xae,0xe5,0x00,0x00,0x00,0x04,0x00,0x01,0x02,0x05]
; GFX13-TRUE16-NEXT: s_wait_samplecnt 0x0 ; encoding: [0x00,0x00,0xc2,0xbf]
; GFX13-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
index 0f547e619f260..5413c6170b5a7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.image.sample.g16.ll
@@ -360,9 +360,8 @@ define amdgpu_ps float @sample_c_d_o_2darray_V1(<8 x i32> inreg %rsrc, <4 x i32>
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V1:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v5, v9, v4, 0x5040100
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v0, [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x4 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
@@ -411,9 +410,8 @@ define amdgpu_ps <2 x float> @sample_c_d_o_2darray_V2(<8 x i32> inreg %rsrc, <4
; GFX12PLUS-TRUE16-LABEL: sample_c_d_o_2darray_V2:
; GFX12PLUS-TRUE16: ; %bb.0: ; %main_body
; GFX12PLUS-TRUE16-NEXT: v_mov_b32_e32 v9, v5
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.l, v4.l
; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
-; GFX12PLUS-TRUE16-NEXT: v_mov_b16_e32 v5.h, v9.l
+; GFX12PLUS-TRUE16-NEXT: v_perm_b32 v5, v9, v4, 0x5040100
; GFX12PLUS-TRUE16-NEXT: image_sample_c_d_o_g16 v[0:1], [v0, v1, v2, v[5:8]], s[0:7], s[8:11] dmask:0x6 dim:SQ_RSRC_IMG_2D_ARRAY
; GFX12PLUS-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12PLUS-TRUE16-NEXT: ; return to shader part epilog
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
index e17e1c621da2f..3e497dfd09575 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.waitcnt.out.order.ll
@@ -86,8 +86,7 @@ define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: sample_load:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
@@ -106,8 +105,7 @@ define amdgpu_ps <3 x float> @sample_load(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX1150-TRUE16-LABEL: sample_load:
; GFX1150-TRUE16: ; %bb.0:
; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX1150-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
@@ -160,8 +158,7 @@ define amdgpu_ps <3 x float> @load_sample(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX11-TRUE16-LABEL: load_sample:
; GFX11-TRUE16: ; %bb.0:
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX11-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX11-TRUE16-NEXT: image_sample_lz v2, [v4, v4], s[0:7], s[8:11] dmask:0x1 dim:SQ_RSRC_IMG_2D
@@ -180,8 +177,7 @@ define amdgpu_ps <3 x float> @load_sample(<8 x i32> inreg %rsrc, <4 x i32> inreg
; GFX1150-TRUE16-LABEL: load_sample:
; GFX1150-TRUE16: ; %bb.0:
; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.l
-; GFX1150-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
+; GFX1150-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040100
; GFX1150-TRUE16-NEXT: v_mov_b32_e32 v4, 0
; GFX1150-TRUE16-NEXT: image_msaa_load v[0:3], v[2:3], s[12:19] dmask:0x1 dim:SQ_RSRC_IMG_2D_MSAA unorm a16
; GFX1150-TRUE16-NEXT: s_waitcnt vmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
index ad9acc26dcba3..e2701809f3090 100644
--- a/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/mad-mix-hi-bf16.ll
@@ -238,10 +238,9 @@ define <2 x bfloat> @mixhi_fptrunc_fadd(float %a, float %b, bfloat %lo) #0 {
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-REAL16-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-REAL16-NEXT: v_perm_b32 v0, v1, v2, 0x5040100
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
.entry:
%add = fadd float %a, %b
@@ -267,10 +266,9 @@ define <2 x bfloat> @mixhi_fptrunc_fadd_f16_src(half %a, float %b, bfloat %lo) #
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-REAL16-NEXT: v_fma_mix_f32 v0, v0, 1.0, v1 op_sel_hi:[1,1,0]
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-REAL16-NEXT: v_perm_b32 v0, v1, v2, 0x5040100
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
.entry:
%a.ext = fpext half %a to float
@@ -297,10 +295,9 @@ define <2 x bfloat> @mixhi_fptrunc_fsub(float %a, float %b, bfloat %lo) #0 {
; GFX1250-REAL16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-REAL16-NEXT: v_sub_f32_e32 v0, v0, v1
-; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1250-REAL16-NEXT: v_cvt_pk_bf16_f32 v1, v0, s0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, v2.l
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX1250-REAL16-NEXT: v_perm_b32 v0, v1, v2, 0x5040100
; GFX1250-REAL16-NEXT: s_set_pc_i64 s[30:31]
.entry:
%sub = fsub float %a, %b
diff --git a/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll b/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
index 4949d917be43e..5af569d03c289 100644
--- a/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
+++ b/llvm/test/CodeGen/AMDGPU/repeated-divisor.ll
@@ -979,10 +979,9 @@ define <6 x half> @v_repeat_divisor_v3f16_x2(<3 x half> %x, <3 x half> %y, <3 x
; GFX11-TRUE16-NEXT: v_pk_mul_f16 v3, v3, v5
; GFX11-TRUE16-NEXT: v_pk_mul_f16 v1, v1, v5
; GFX11-TRUE16-NEXT: v_pk_mul_f16 v0, v0, v4
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: v_repeat_divisor_v3f16_x2:
diff --git a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
index 3bbc5b9113672..726f78b89eb82 100644
--- a/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
+++ b/llvm/test/CodeGen/AMDGPU/scalar_to_vector.ll
@@ -474,8 +474,7 @@ define i64 @bitcast_combine_scalar_to_vector_v4i16(i16 %arg) {
; GFX11-NEXT: v_lshrrev_b16 v1.l, 8, v1.l
; GFX11-NEXT: v_or_b16 v2.h, v1.l, v1.h
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-NEXT: v_mov_b16_e32 v1.h, v2.h
+; GFX11-NEXT: v_perm_b32 v1, v2, v2, 0x7060302
; GFX11-NEXT: v_mov_b32_e32 v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
%arg.cast = bitcast i16 %arg to <2 x i8>
diff --git a/llvm/test/CodeGen/AMDGPU/si-post-ra-merge-v-mov-b16.mir b/llvm/test/CodeGen/AMDGPU/si-post-ra-merge-v-mov-b16.mir
new file mode 100644
index 0000000000000..d650f8b53fa47
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/si-post-ra-merge-v-mov-b16.mir
@@ -0,0 +1,411 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 -run-pass si-post-ra-16bit-mov-folding -verify-machineinstrs -o - %s | FileCheck %s
+# RUN: llc -mtriple=amdgcn -mcpu=gfx1100 -debugify-and-strip-all-safe -mattr=+real-true16 -run-pass si-post-ra-16bit-mov-folding -verify-machineinstrs -o - %s | FileCheck %s
+
+# Tests for mergeTrue16MovB16Pair: pairs of v_mov_b16 targeting lo16 and hi16
+# of the same VGPR are merged into a single 32-bit instruction (true16 mode).
+
+---
+# Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.h
+# => v_lshrrev_b32 v0, 16, v2
+name: merge_hi_zero_lo_from_hi
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: merge_hi_zero_lo_from_hi
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e32 16, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+name: merge_hi_zero_lo_from_hi_vop3
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr128
+
+ ; CHECK-LABEL: name: merge_hi_zero_lo_from_hi_vop3
+ ; CHECK: liveins: $vgpr128
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_LSHRREV_B32_e64 16, $vgpr128, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e64 0, $vgpr128_hi16, 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l
+# => v_lshlrev_b32 v0, 16, v2
+name: merge_lo_zero_hi_from_lo
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: merge_lo_zero_hi_from_lo
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_LSHLREV_B32_e32 16, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, s2
+# => v_lshlrev_b32 v0, 16, s2
+name: merge_lo_zero_hi_from_lo_sgpr
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr2
+
+ ; CHECK-LABEL: name: merge_lo_zero_hi_from_lo_sgpr
+ ; CHECK: liveins: $sgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_LSHLREV_B32_e64 16, $sgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $sgpr2, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
+# => v_and_b32 v0, 0xffff0000, v2
+name: merge_lo_zero_hi_from_hi
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: merge_lo_zero_hi_from_hi
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_AND_B32_e32 4294901760, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l
+# => v_and_b32 v0, 0x0000ffff, v2
+name: merge_hi_zero_lo_from_lo
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: merge_hi_zero_lo_from_lo
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_AND_B32_e32 65535, $vgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, s2
+# => v_and_b32 v0, 0x0000ffff, s2
+name: merge_hi_zero_lo_from_lo_sgpr
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr2
+
+ ; CHECK-LABEL: name: merge_hi_zero_lo_from_lo_sgpr
+ ; CHECK: liveins: $sgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_AND_B32_e64 65535, $sgpr2, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $sgpr2, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 0, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.l
+name: merge_lo_from_lo_hi_from_lo
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_lo_from_lo_hi_from_lo
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr3, $vgpr2, 84148480, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_lo16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.h
+name: merge_lo_from_hi_hi_from_hi
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_lo_from_hi_hi_from_hi
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr3, $vgpr2, 117834498, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, v3.h
+name: merge_lo_from_lo_hi_from_hi
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_lo_from_lo_hi_from_hi
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr3, $vgpr2, 117833984, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, v2.h + v_mov_b16 v0.h, v3.l
+name: merge_lo_from_hi_hi_from_lo
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_lo_from_hi_hi_from_lo
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr3, $vgpr2, 84148994, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_lo16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, v2.l + v_mov_b16 v0.h, s3
+name: merge_lo_from_lo_hi_from_lo_1_sgpr
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $sgpr3
+
+ ; CHECK-LABEL: name: merge_lo_from_lo_hi_from_lo_1_sgpr
+ ; CHECK: liveins: $vgpr2, $sgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $sgpr3, $vgpr2, 84148480, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $sgpr3, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Pattern: v_mov_b16 v0.l, s2 + v_mov_b16 v0.h, s3
+# Reject, constant bus violation
+name: merge_lo_from_lo_hi_from_lo_2_sgpr
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $sgpr2, $sgpr3
+
+ ; CHECK-LABEL: name: merge_lo_from_lo_hi_from_lo_2_sgpr
+ ; CHECK: liveins: $sgpr2, $sgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e32 $sgpr2, implicit $exec
+ ; CHECK-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e32 $sgpr3, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $sgpr2, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $sgpr3, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Negative: intervening instruction that clobbers dst — must NOT merge.
+name: no_merge_dst_clobbered
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2
+
+ ; CHECK-LABEL: name: no_merge_dst_clobbered
+ ; CHECK: liveins: $vgpr2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr0 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr2_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+name: merge_src_clobbered_First_Src
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_src_clobbered_First_Src
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr3, $vgpr2, 117833984, implicit $exec
+ ; CHECK-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+name: merge_src_clobbered_Second_Src
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_src_clobbered_Second_Src
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr3, $vgpr2, 117833984, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+name: merge_src_read_First_dst
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_src_read_First_dst
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr3, $vgpr2, 117833984, implicit $exec
+ ; CHECK-NEXT: $vgpr4_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr4_lo16 = V_MOV_B16_t16_e32 $vgpr0_lo16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+name: merge_src_read_Second_dst
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_src_read_Second_dst
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0_hi16 = V_AND_B16_t16_e64 0, 0, 0, 0, 0, implicit $exec
+ ; CHECK-NEXT: $vgpr4_lo16 = V_MOV_B16_t16_e32 $vgpr0_hi16, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr3, $vgpr2, 117833984, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_hi16 = V_AND_B16_t16_e64 0, 0, 0, 0, 0, implicit $exec
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr4_lo16 = V_MOV_B16_t16_e32 $vgpr0_hi16, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+name: merge_src_read_dst_declined
+# Negative: intervening instruction that read dst — must NOT merge.
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_src_read_dst_declined
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ ; CHECK-NEXT: $vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec
+ ; CHECK-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr4 = V_MOV_B32_e32 $vgpr0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+name: merge_src_clobbered_both_accepted
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: merge_src_clobbered_both_accepted
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $vgpr0 = V_PERM_B32_e64 $vgpr3, $vgpr2, 117833984, implicit $exec
+ ; CHECK-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
+
+---
+# Negative: intervening instruction that clobbers both src — must NOT merge.
+name: no_merge_src_clobbered_both_declined
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $vgpr2, $vgpr3
+
+ ; CHECK-LABEL: name: no_merge_src_clobbered_both_declined
+ ; CHECK: liveins: $vgpr2, $vgpr3
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ ; CHECK-NEXT: $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ ; CHECK-NEXT: $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ ; CHECK-NEXT: S_ENDPGM 0, implicit $vgpr0
+ $vgpr0_lo16 = V_MOV_B16_t16_e32 $vgpr2_lo16, implicit $exec
+ $vgpr2 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr3 = V_MOV_B32_e32 0, implicit $exec
+ $vgpr0_hi16 = V_MOV_B16_t16_e32 $vgpr3_hi16, implicit $exec
+ S_ENDPGM 0, implicit $vgpr0
+...
diff --git a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
index a347e965fd472..431dca2f47c48 100644
--- a/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/sub.v2i16.ll
@@ -795,9 +795,8 @@ define amdgpu_kernel void @v_test_sub_v2i16_zext_to_v2i64(ptr addrspace(1) %out,
; GFX11-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
; GFX11-TRUE16-NEXT: s_mov_b32 s2, -1
; GFX11-TRUE16-NEXT: v_pk_sub_i16 v0, v2, v0
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.h, v1.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v2.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_perm_b32 v2, v1, v0, 0x5040302
; GFX11-TRUE16-NEXT: v_cvt_u32_u16_e32 v0, v0.l
; GFX11-TRUE16-NEXT: buffer_store_b128 v[0:3], off, s[0:3], 0
; GFX11-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
index 297d43357c376..30cf920631342 100644
--- a/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
+++ b/llvm/test/CodeGen/AMDGPU/v_swap_b16.ll
@@ -8,18 +8,17 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX11-TRUE16-LABEL: swap:
; GFX11-TRUE16: ; %bb.0: ; %entry
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
; GFX11-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX11-TRUE16-NEXT: .LBB0_1: ; %loop
; GFX11-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX11-TRUE16-NEXT: v_swap_b16 v0.l, v0.h
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX11-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: s_and_not1_b32 exec_lo, exec_lo, s0
; GFX11-TRUE16-NEXT: s_cbranch_execnz .LBB0_1
; GFX11-TRUE16-NEXT: ; %bb.2: ; %ret
@@ -52,15 +51,15 @@ define half @swap(half %a, half %b, i32 %i) {
; GFX12-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.h, v0.l
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
; GFX12-TRUE16-NEXT: s_mov_b32 s0, 0
; GFX12-TRUE16-NEXT: .LBB0_1: ; %loop
; GFX12-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX12-TRUE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
; GFX12-TRUE16-NEXT: v_swap_b16 v0.l, v0.h
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX12-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v2
; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-TRUE16-NEXT: s_or_b32 s0, vcc_lo, s0
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index 30b0554fcc482..efe671602759e 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -323,25 +323,14 @@ define <4 x half> @shuffle_v4f16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4f16_35u5:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4f16_35u5:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4f16_35u5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: global_load_b32 v1, v[2:3], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 3, i32 5, i32 poison, i32 5>
@@ -998,8 +987,7 @@ define <4 x half> @shuffle_v4f16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1058,8 +1046,7 @@ define <4 x half> @shuffle_v4f16_5623(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1106,12 +1093,9 @@ define <4 x half> @shuffle_v4f16_3456(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040302
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_3456:
@@ -1158,12 +1142,9 @@ define <4 x half> @shuffle_v4f16_5634(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v0, 0x5040302
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_5634:
@@ -1223,11 +1204,9 @@ define <4 x half> @shuffle_v4f16_5734(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v0, 0x5040302
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1288,8 +1267,7 @@ define <4 x i16> @shuffle_v4i16_2356(ptr addrspace(1) %arg0, ptr addrspace(1) %a
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1423,8 +1401,7 @@ define <4 x half> @shuffle_v4f16_1010(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v1, 0x5040302
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -1481,8 +1458,7 @@ define <4 x half> @shuffle_v4f16_1100(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v0, v0, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1587,22 +1563,13 @@ define <4 x half> @shuffle_v4f16_2333(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4f16_2333:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4f16_2333:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4f16_2333:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -1636,22 +1603,13 @@ define <4 x half> @shuffle_v4f16_6667(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4f16_6667:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4f16_6667:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4f16_6667:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x half>, ptr addrspace(1) %arg0
%val1 = load <4 x half>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x half> %val0, <4 x half> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -1829,8 +1787,7 @@ define <4 x half> @shuffle_v8f16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrspace
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -1919,8 +1876,7 @@ define <4 x half> @shuffle_v2f16_0122(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v0, v0, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v2f16_0122:
@@ -2147,8 +2103,7 @@ define <4 x half> @shuffle_v4f16_0456(ptr addrspace(1) %arg0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4f16_0456:
@@ -2413,10 +2368,8 @@ define <2 x half> @hi16low16bits_v2bf16(ptr addrspace(1) %x0, ptr addrspace(1) %
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: hi16low16bits_v2bf16:
@@ -2569,10 +2522,8 @@ define <2 x i16> @i16_hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1)
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: i16_hi16low16bits:
@@ -3613,25 +3564,14 @@ define <4 x bfloat> @shuffle_v4bf16_35u5(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_mov_b32_e32 v1, v4
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4bf16_35u5:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4bf16_35u5:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: global_load_b32 v1, v[2:3], off
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4bf16_35u5:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: global_load_b32 v1, v[2:3], off
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 3, i32 5, i32 poison, i32 5>
@@ -4288,8 +4228,7 @@ define <4 x bfloat> @shuffle_v4bf16_2356(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4348,8 +4287,7 @@ define <4 x bfloat> @shuffle_v4bf16_5623(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4396,12 +4334,9 @@ define <4 x bfloat> @shuffle_v4bf16_3456(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040302
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v1, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_3456:
@@ -4448,12 +4383,9 @@ define <4 x bfloat> @shuffle_v4bf16_5634(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v0, 0x5040302
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_5634:
@@ -4513,11 +4445,9 @@ define <4 x bfloat> @shuffle_v4bf16_5734(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v3.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v2, v0, 0x5040302
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, v3
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -4617,8 +4547,7 @@ define <4 x bfloat> @shuffle_v4bf16_1010(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[1:2], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v1.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v1, 0x5040302
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, v0
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
@@ -4675,8 +4604,7 @@ define <4 x bfloat> @shuffle_v4bf16_1100(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v0, v0, 0x5040100
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -4781,22 +4709,13 @@ define <4 x bfloat> @shuffle_v4bf16_2333(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4bf16_2333:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4bf16_2333:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4bf16_2333:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -4830,22 +4749,13 @@ define <4 x bfloat> @shuffle_v4bf16_6667(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX10-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
-; GFX11-TRUE16-LABEL: shuffle_v4bf16_6667:
-; GFX11-TRUE16: ; %bb.0:
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.h
-; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
-;
-; GFX11-FAKE16-LABEL: shuffle_v4bf16_6667:
-; GFX11-FAKE16: ; %bb.0:
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX11-FAKE16-NEXT: global_load_b32 v0, v[0:1], off offset:4
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
-; GFX11-FAKE16-NEXT: s_setpc_b64 s[30:31]
+; GFX11-LABEL: shuffle_v4bf16_6667:
+; GFX11: ; %bb.0:
+; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX11-NEXT: global_load_b32 v0, v[0:1], off offset:4
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_perm_b32 v1, v0, v0, 0x7060302
+; GFX11-NEXT: s_setpc_b64 s[30:31]
%val0 = load <4 x bfloat>, ptr addrspace(1) %arg0
%val1 = load <4 x bfloat>, ptr addrspace(1) %arg1
%shuffle = shufflevector <4 x bfloat> %val0, <4 x bfloat> %val1, <4 x i32> <i32 2, i32 3, i32 3, i32 3>
@@ -5023,8 +4933,7 @@ define <4 x bfloat> @shuffle_v8bf16_13_14_2_3(ptr addrspace(1) %arg0, ptr addrsp
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v[2:3], off offset:8
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[0:1], off offset:4
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
@@ -5113,8 +5022,7 @@ define <4 x bfloat> @shuffle_v2bf16_0122(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v0.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v0.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v0, v0, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v2bf16_0122:
@@ -5704,8 +5612,7 @@ define <4 x bfloat> @shuffle_v4bf16_0456(ptr addrspace(1) %arg0, ptr addrspace(1
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v[0:1], off
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v2.l
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, v2.h
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.h, v3.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v1, v3, v2, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: shuffle_v4bf16_0456:
@@ -5912,10 +5819,8 @@ define <2 x bfloat> @hi16low16bits(ptr addrspace(1) %x0, ptr addrspace(1) %x1) {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-TRUE16-NEXT: global_load_b32 v0, v[0:1], off
; GFX11-TRUE16-NEXT: global_load_b32 v1, v[2:3], off
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.l, v0.h
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.l
+; GFX11-TRUE16-NEXT: v_perm_b32 v0, v1, v0, 0x5040302
; GFX11-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-FAKE16-LABEL: hi16low16bits:
More information about the llvm-commits
mailing list