[llvm] [AMDGPU] merge 16bit mov pairs in post-RA peephole (PR #208625)
Jay Foad via llvm-commits
llvm-commits at lists.llvm.org
Thu Jul 16 00:22:04 PDT 2026
================
@@ -763,6 +771,263 @@ MachineInstrBuilder SIPreEmitPeephole::createUnpackedMI(MachineInstr &I,
return NewMI;
}
+// Helper: extract the src operand and whether it is from the hi16 half.
+// Post-RA, both V_MOV_B16_t16_e32 and V_MOV_B16_t16_e64 use VGPR_16 dst
+// physical registers whose encoding already encodes hi/lo (IS_HI16 bit).
+void SIPreEmitPeephole::getMovB16Info(const MachineInstr &MI,
+ const SIRegisterInfo *TRI,
+ MCRegister &SrcReg16, bool &SrcIsVGPR,
+ MCRegister &SrcReg32, bool &SrcIsHi,
+ bool &SrcIsImm, int64_t &ImmVal) const {
+ SrcIsImm = false;
+ SrcIsHi = false;
+ SrcIsVGPR = false;
+ SrcReg16 = MCRegister();
+ SrcReg32 = MCRegister();
+
+ const MachineOperand *SrcOp = TII->getNamedOperand(MI, AMDGPU::OpName::src0);
+
+ if (SrcOp->isImm()) {
+ SrcIsImm = true;
+ ImmVal = SrcOp->getImm();
+ return;
+ }
+
+ SrcReg16 = SrcOp->getReg().asMCReg();
+ SrcIsVGPR = AMDGPU::VGPR_16RegClass.contains(SrcReg16);
+ if (SrcIsVGPR) {
+ SrcIsHi = AMDGPU::isHi16Reg(SrcReg16, *TRI);
+ SrcReg32 = TRI->get32BitRegister(SrcReg16);
+ } else {
+ SrcIsHi = false;
+ SrcReg32 = SrcReg16;
+ }
+}
+
+// clang-format off
+// Try to merge a pair of v_mov_b16 instructions targeting the lo16 and hi16
+// halves of the same VGPR into a single 32-bit instruction.
+//
+// Caller guarantee the pair to be two v_mov_b16 and targets the same dst32
+//
+// Patterns:
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.l/s2 => v_and_b32 v0,0xffff,v2/s2
+// v_mov_b16 v0.h, 0 v_mov_b16 v0.l, v2.h => v_lshrrev_b32 v0,16,v2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.l/s2 => v_lshlrev_b32 v0,16,v2/s2
+// v_mov_b16 v0.l, 0 v_mov_b16 v0.h, v2.h => v_and_b32 v0,0xffff0000,v2
+// v_mov_b16 v0.l, v.x/s v_mov_b16 v0.h, v.y/s => v_pack_b32_f16 v0, v/s, v/s
+// clang-format on
+bool SIPreEmitPeephole::mergeSingleMovB16Pair(MachineInstr &Lo,
+ MachineInstr &Hi,
+ bool IsHiFirst) const {
+ // Lo and Hi share the same Dst32
+ MCRegister LoDst = Lo.getOperand(0).getReg().asMCReg();
+ MCRegister Dst32 = TRI->get32BitRegister(LoDst);
+
+ // Extract source info for Lo and Hi.
+ MCRegister LoSrc16, LoSrc32, HiSrc16, HiSrc32;
+ bool LoSrcIsHi, HiSrcIsHi, LoSrcIsImm, HiSrcIsImm, LoSrcIsVGPR, HiSrcIsVGPR;
+ int64_t LoImm = 0, HiImm = 0;
+
+ getMovB16Info(Lo, TRI, LoSrc16, LoSrcIsVGPR, LoSrc32, LoSrcIsHi, LoSrcIsImm,
+ LoImm);
+ getMovB16Info(Hi, TRI, HiSrc16, HiSrcIsVGPR, HiSrc32, HiSrcIsHi, HiSrcIsImm,
+ HiImm);
+
+ MachineInstr &FirstMI = IsHiFirst ? Hi : Lo;
+ MachineInstr &SecondMI = IsHiFirst ? Lo : Hi;
+
+ // Data Conflict counter
+ auto UpperBound = SecondMI.getIterator();
+ auto LowerBound = FirstMI.getIterator();
+ unsigned LoopCnt = 0, UpperBoundCnt = UINT_MAX, LowerBoundCnt = 0;
+
+ MachineBasicBlock &MBB = *Lo.getParent();
+
+ // Check that between Lo and Hi, there are no instructions that:
+ // - modify Dst32
+ // - modify LoSrc16 or HiSrc16 depending on order (data dependency)
+ // We scan from the instruction after the first mov up to (but not including)
+ // the second mov.
+ MCRegister FirstSrc16 = IsHiFirst ? HiSrc16 : LoSrc16;
+ MCRegister SecondSrc16 = IsHiFirst ? LoSrc16 : HiSrc16;
+ for (MachineInstr &Scan :
+ drop_begin(make_range(FirstMI.getIterator(), SecondMI.getIterator()))) {
+ if (Scan.modifiesRegister(Dst32, TRI))
+ return false;
+ LoopCnt++;
+ if (FirstSrc16 && LoopCnt < UpperBoundCnt &&
+ Scan.modifiesRegister(FirstSrc16, TRI)) {
+ UpperBound = Scan.getIterator();
+ UpperBoundCnt = LoopCnt;
+ }
+ if (SecondSrc16 && LoopCnt > LowerBoundCnt &&
+ Scan.modifiesRegister(SecondSrc16, TRI)) {
+ LowerBound = Scan.getIterator();
+ LowerBoundCnt = LoopCnt;
+ }
+ }
+
+ // No spot maintains data dependency
+ if (LowerBoundCnt >= UpperBoundCnt)
+ return false;
+
+ // Insert MI before selected. Any spots between (LowerBound, UpperBound] would
+ // work
+ MachineInstr &Selected = *(++LowerBound);
+ const DebugLoc &DL = Selected.getDebugLoc();
+
+ // Now match patterns and emit the replacement instruction.
+ // Insert on Selected MI location, then remove both mov.
+
+ // Pattern: v_mov_b16 v0.l, v2.x/s2 + v_mov_b16 v0.h, v3.y/s3
+ // => v_pack_b32_f16 v0,v2.x/s2,v3.y/s3
+ if (!HiSrcIsImm && !LoSrcIsImm) {
+ BuildMI(MBB, Selected, DL, TII->get(AMDGPU::V_PACK_B32_F16_t16_e64), Dst32)
+ .addImm(0) // SrcMod
+ .addReg(LoSrc16)
+ .addImm(0) // SrcMod
+ .addReg(HiSrc16)
+ .addImm(0) // Clamp
+ .addImm(0); // Opsel
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ bool Usevop2 =
+ AMDGPU::VGPR_32_Lo128RegClass.contains(Dst32) &&
+ (LoSrcIsImm ||
+ (LoSrcIsVGPR && AMDGPU::VGPR_32_Lo128RegClass.contains(LoSrc32))) &&
+ (HiSrcIsImm ||
+ (HiSrcIsVGPR && AMDGPU::VGPR_32_Lo128RegClass.contains(HiSrc32)));
+
+ // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.l/s2
+ // => v_and_b32 v0, 0x0000ffff, v2/s2
+ if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && !LoSrcIsHi) {
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_AND_B32_e32 : AMDGPU::V_AND_B32_e64),
+ Dst32)
+ .addImm(0x0000ffff)
+ .addReg(LoSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.h, 0 + v_mov_b16 v0.l, v2.h
+ // => v_lshrrev_b32 v0, 16, v2
+ if (HiSrcIsImm && HiImm == 0 && !LoSrcIsImm && LoSrcIsHi) {
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_LSHRREV_B32_e32
+ : AMDGPU::V_LSHRREV_B32_e64),
+ Dst32)
+ .addImm(16)
+ .addReg(LoSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.l/s2
+ // => v_lshlrev_b32 v0, 16, v2/s2
+ if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && !HiSrcIsHi) {
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_LSHLREV_B32_e32
+ : AMDGPU::V_LSHLREV_B32_e64),
+ Dst32)
+ .addImm(16)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ // Pattern: v_mov_b16 v0.l, 0 + v_mov_b16 v0.h, v2.h
+ // => v_and_b32 v0, 0xffff0000, v2
+ if (LoSrcIsImm && LoImm == 0 && !HiSrcIsImm && HiSrcIsHi) {
+ BuildMI(MBB, Selected, DL,
+ TII->get(Usevop2 ? AMDGPU::V_AND_B32_e32 : AMDGPU::V_AND_B32_e64),
+ Dst32)
+ .addImm(0xffff0000)
+ .addReg(HiSrc32);
+ Lo.eraseFromParent();
+ Hi.eraseFromParent();
+ return true;
+ }
+
+ return false;
+}
+
+// Merge pairs of v_mov_b16 targeting the lo16 and hi16 halves of the same
+// VGPR into a single 32-bit instruction (true16 mode only).
+bool SIPreEmitPeephole::mergeMovB16Pairs(MachineFunction &MF) const {
+ bool Changed = false;
+ for (MachineBasicBlock &MBB : MF) {
+ // Map from 32-bit VGPR to the pending v_mov_b16 and its age.
+ // Age tracks how many non-mov-b16 instructions have passed since the
+ // 16-bit write, used to bound the search window.
+ struct Pending {
+ MCRegister Dst32;
+ MachineInstr *MI;
+ unsigned IsHi;
+ };
+ // Search window size
+ const unsigned ScanLimit = 16;
+ std::array<Pending, ScanLimit> CirBuf = {{{MCRegister(), nullptr, false}}};
+ SmallDenseMap<MCRegister, unsigned> PendingWrites;
+ unsigned Head = 0;
+
+ for (auto &MI : make_early_inc_range(MBB)) {
----------------
jayfoad wrote:
No auto
https://github.com/llvm/llvm-project/pull/208625
More information about the llvm-commits
mailing list