[llvm] [AMDGPU] merge 16bit mov pairs in post-RA peephole (PR #208625)
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 04:11:22 PDT 2026
================
@@ -763,6 +766,229 @@ MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
return NewMI;
}
+// Helper: extract the src operand and whether it is from the hi16 half.
+// Post-RA, both V_MOV_B16_t16_e32 and V_MOV_B16_t16_e64 use VGPR_16 physical
+// registers whose encoding already encodes hi/lo (IS_HI16 bit).
+// Returns false if the source is not a physical VGPR_16 or immediate zero.
+//
+// Operand layouts (post-RA, physical registers):
+// V_MOV_B16_t16_e32: dst(0), src0(1)
+// V_MOV_B16_t16_e64: dst(0), src0_mods(1), src0(2), op_sel(3)
+static bool getMovB16Info(const MachineInstr &MI, const SIRegisterInfo *TRI,
+ MCRegister &SrcReg16, MCRegister &SrcReg32,
+ bool &SrcIsHi, bool &SrcIsImm, int64_t &ImmVal) {
+ SrcIsImm = false;
+ SrcIsHi = false;
+ SrcReg16 = MCRegister();
+ SrcReg32 = MCRegister();
+
+ unsigned Opc = MI.getOpcode();
+ const MachineOperand *SrcOp = nullptr;
+
+ if (Opc == AMDGPU::V_MOV_B16_t16_e64)
+ SrcOp = &MI.getOperand(2);
+ else if (Opc == AMDGPU::V_MOV_B16_t16_e32)
+ SrcOp = &MI.getOperand(1);
+ else
+ return false;
+
+ if (SrcOp->isImm()) {
+ SrcIsImm = true;
+ ImmVal = SrcOp->getImm();
+ return true;
+ }
+
+ if (!SrcOp->isReg() || !SrcOp->getReg().isPhysical())
+ return false;
+
+ MCRegister SrcReg = SrcOp->getReg().asMCReg();
+
+ // We require the source to be a 16-bit VGPR so we can determine hi/lo.
+ if (!AMDGPU::VGPR_16RegClass.contains(SrcReg))
+ return false;
+
+ SrcIsHi = AMDGPU::isHi16Reg(SrcReg, *TRI);
+ SrcReg16 = SrcReg;
+ SrcReg32 = TRI->get32BitRegister(SrcReg);
+ return SrcReg32.isValid();
+}
+
+// clang-format off
+// Try to merge a pair of v_mov_b16 instructions targeting the lo16 and hi16
+// halves of the same VGPR into a single 32-bit instruction.
+//
+// Caller guarantee the pair to be two v_mov_b16 and targets the same dst32
+//
+// Patterns:
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.l => v_and_b32 v0,0xffff,v2
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.h => v_lshrrev_b32 v0,16,v2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.l => v_lshlrev_b32 v0,16,v2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.h => v_and_b32 v0,0xffff0000,v2
+// v_mov_b16 v0.l, v2.x v_mov_b16 v0.h, v3.y => v_pack_b32_f16 v0,v2.x,v3.y
+// clang-format on
+bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
+ MachineInstr &Hi,
+ bool IsHiFirst) const {
+ // Lo and Hi share the same Dst32
+ MCRegister LoDst = Lo.getOperand(0).getReg().asMCReg();
+ MCRegister Dst32 = TRI->get32BitRegister(LoDst);
+
+ // Extract source info for Lo and Hi.
+ MCRegister LoSrc16, LoSrc32, HiSrc16, HiSrc32;
+ bool LoSrcIsHi, HiSrcIsHi, LoSrcIsImm, HiSrcIsImm;
+ int64_t LoImm = 0, HiImm = 0;
+
+ if (!getMovB16Info(Lo, TRI, LoSrc16, LoSrc32, LoSrcIsHi, LoSrcIsImm, LoImm))
+ return false;
+ if (!getMovB16Info(Hi, TRI, HiSrc16, HiSrc32, HiSrcIsHi, HiSrcIsImm, HiImm))
+ return false;
+
+ MachineInstr &FirstMI = IsHiFirst ? Hi : Lo;
+ MachineInstr &SecondMI = IsHiFirst ? Lo : Hi;
+ bool IsSecondImm = IsHiFirst ? LoSrcIsImm : HiSrcIsImm;
+
+ MachineBasicBlock &MBB = *FirstMI.getParent();
+ const DebugLoc &DL = FirstMI.getDebugLoc();
+
+ // Check that between Lo and Hi, there are no instructions that:
+ // - modify Dst32 (except through Lo/Hi themselves)
+ // - modify LoSrc16 or HiSrc16 dependinig on order (data dependency)
+ // We scan from the instruction after the first mov up to (but not including)
+ // the second mov.
+ MCRegister SecondSrc16 = IsHiFirst ? LoSrc16 : HiSrc16;
+ for (auto &It :
+ drop_begin(make_range(FirstMI.getIterator(), SecondMI.getIterator()))) {
+ const MachineInstr &Scan = It;
+ if (Scan.modifiesRegister(Dst32, TRI))
+ return false;
+ if (!IsSecondImm && Scan.modifiesRegister(SecondSrc16, TRI))
+ return false;
+ }
+
+ // Now match patterns and emit the replacement instruction.
+ // Insert before the first (Lo) instruction, then remove both.
+
+ // Pattern: v_mov_b16 v0.l, v2.x + v_mov_b16 v0.h, v3.y
+ // => v_pack_b32_f16 v0,v2.x,v3.y
+ if (!HiSrcIsImm && !LoSrcIsImm) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_PACK_B32_F16_t16_e64), Dst32)
+ .addImm(0) // SrcMod
+ .addReg(LoSrc16)
+ .addImm(0) // SrcMod
+ .addReg(HiSrc16)
+ .addImm(0) // Clamp
+ .addImm(0); // Opsel
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l
+ // => v_and_b32 v0, 0x0000ffff, v2
+ if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && !LoSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e32), Dst32)
+ .addImm(0x0000ffff)
+ .addReg(LoSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.h
+ // => v_lshrrev_b32 v0, 16, v2
+ if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && LoSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHRREV_B32_e32), Dst32)
+ .addImm(16)
+ .addReg(LoSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l
+ // => v_lshlrev_b32 v0, 16, v2
+ if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && !HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_LSHLREV_B32_e32), Dst32)
+ .addImm(16)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
+ // => v_and_b32 v0, 0xffff0000, v2
+ if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && HiSrcIsHi) {
+ BuildMI(MBB, Lo, DL, TII->get(AMDGPU::V_AND_B32_e32), Dst32)
+ .addImm(0xffff0000)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ return false;
+}
+
+// Merge pairs of v_mov_b16 targeting the lo16 and hi16 halves of the same
+// VGPR into a single 32-bit instruction (true16 mode only).
+bool SIPreEmitPeephole::mergeMovB16Pair(MachineFunction &MF) const {
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ // Map from 32-bit VGPR to the pending v_mov_b16 and its age.
+ // Age tracks how many non-mov-b16 instructions have passed since the
+ // lo16 write, used to bound the search window.
+ struct Pending {
+ MachineInstr *MI;
+ unsigned Age; // instructions since was seen
+ unsigned IsHi;
+ };
+ // Search window size
+ const unsigned ScanLimit = 16;
+ SmallDenseMap<MCRegister, Pending> PendingWrites;
+
+ for (auto &MI : make_early_inc_range(MBB)) {
+ unsigned Opc = MI.getOpcode();
+ bool IsMovB16 = (Opc == AMDGPU::V_MOV_B16_t16_e32 ||
+ Opc == AMDGPU::V_MOV_B16_t16_e64);
+
+ if (!IsMovB16) {
+ // Age all pending lo writes and invalidate stale or clobbered ones.
+ for (auto &[key, value] : PendingWrites)
+ value.Age++;
+
+ PendingWrites.remove_if([&](const auto &KV) {
+ return (KV.second.Age >= ScanLimit ||
+ MI.modifiesRegister(KV.first, TRI));
+ });
+ continue;
+ }
+
+ MCRegister DstReg = MI.getOperand(0).getReg().asMCReg();
+
+ bool DstIsHi = AMDGPU::isHi16Reg(DstReg, *TRI);
+ MCRegister Dst32 = TRI->get32BitRegister(DstReg);
+ if (!Dst32.isValid())
----------------
jayfoad wrote:
How can this fail?
https://github.com/llvm/llvm-project/pull/208625
More information about the llvm-commits
mailing list