[llvm] [AMDGPU][CodeGen] Post-RA VGPR MSB group optimization pass (PR #222666)
Lucas Ramirez via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 2 07:53:46 PDT 2026
https://github.com/lucas-rami updated https://github.com/llvm/llvm-project/pull/222666
>From 011825b3021b9a7d31131471e6da2546d6d75069 Mon Sep 17 00:00:00 2001
From: Lucas Ramirez <lucas.rami at proton.me>
Date: Thu, 10 Sep 2026 13:52:27 +0000
Subject: [PATCH 1/5] [AMDGPU][CodeGen] Post-RA MSB group optimization pass
This pass is meant to run between register allocation and
virtual-to-physical
register rewriting. On subtargets where the MSBs of some
VGPRs come from the processor's MODE, the pass tries to modify the
existing
virtual-to-physical register mappings to reduce the number of
MODE-setting
instructions that will need to be inserted in the program to honor MSB
group
differences between physical VGPRs. The pass cannot cause extra spilling
to
occur.
In the future, the intent is for this pass to also try to minimize VGPR
bank
conflicts on subtarget where it is relevant.
---
llvm/lib/Target/AMDGPU/AMDGPU.h | 3 +
.../AMDGPU/AMDGPUOptimizeVGPREncoding.cpp | 1835 +++++++++++++++++
.../AMDGPU/AMDGPUOptimizeVGPREncoding.h | 23 +
llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def | 1 +
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 13 +
llvm/lib/Target/AMDGPU/CMakeLists.txt | 1 +
6 files changed, 1876 insertions(+)
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
create mode 100644 llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.h
diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h
index 809540bd05b45c..c5805c4d7f1354 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPU.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPU.h
@@ -614,6 +614,9 @@ extern char &AMDGPUInsertDelayAluID;
void initializeAMDGPULowerVGPREncodingLegacyPass(PassRegistry &);
extern char &AMDGPULowerVGPREncodingLegacyID;
+void initializeAMDGPUOptimizeVGPREncodingLegacyPass(PassRegistry &);
+extern char &SIAMDGPUOptimizeVGPREncodingLegacyID;
+
void initializeSIInsertHardClausesLegacyPass(PassRegistry &);
extern char &SIInsertHardClausesID;
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
new file mode 100644
index 00000000000000..f19911ce6b797a
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
@@ -0,0 +1,1835 @@
+//===-- AMDGPUOptimizeVGPREncoding.cpp --------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This pass is meant to run between register allocation and
+/// virtual-to-physical register rewriting. On subtargets where the MSBs of some
+/// VGPRs come from the processor's MODE, the pass tries to modify the existing
+/// virtual-to-physical register mappings to reduce the number of MODE-setting
+/// instructions that will need to be inserted in the program to honor MSB group
+/// differences between physical VGPRs. The pass cannot cause extra spilling to
+/// occur.
+///
+/// In the future, the intent is for this pass to also try to minimize VGPR bank
+/// conflicts on subtarget where it is relevant.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUOptimizeVGPREncoding.h"
+#include "AMDGPU.h"
+#include "GCNSubtarget.h"
+#include "SIInstrInfo.h"
+#include "SIRegisterInfo.h"
+#include "Utils/AMDGPUBaseInfo.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveRegMatrix.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/CodeGen/RegisterClassInfo.h"
+#include "llvm/CodeGen/VirtRegMap.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/MC/MCRegister.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#include <cstring>
+#include <functional>
+#include <string>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "amdgpu-optimize-vgpr-encoding"
+
+namespace {
+
+/// MSB group, identified by an unsigned ID in [0, NumMSBGroups).
+using MSBGroup = unsigned;
+static constexpr unsigned NumMSBGroups = 4;
+static constexpr unsigned DefaultGroup = 0;
+
+/// Operand type where the MSB group is relevant, identified by an unsigned ID
+/// in [0, NumOprdTypes).
+using OprdType = unsigned;
+static constexpr unsigned NumOprdTypes = 4;
+
+/// An instruction that has at least one VGPR operand whose MSBs are provided by
+/// MODE. Instructions are part of a list and refer to other "neighbor"
+/// instructions through their respective index in this list. Two instructions
+/// are neighbors if they have at least one VGPR operand in the same operand
+/// type and no other instruction with such an operand in between them. Two
+/// neighbor instructions whose respective VGPR in a shared operand type differ
+/// in MSB group require at least one MODE-setting instruction to be placed
+/// somewhere in between them.
+struct ModeInstr {
+ /// Sentinel value in previous/next index arrays to indicate the non-existence
+ /// of a previous/next instruction.
+ static constexpr unsigned NoIdx = ~0U;
+
+ /// For each operand type, virtual of physical VGPR operand used by the
+ /// instruction. A null register indicates the instruction has no VGPR operand
+ /// of that type. For VOPD instructions, this holds VGPRs for the first of the
+ /// two instruction which define a VGPR of each operand type.
+ std::array<Register, NumOprdTypes> Oprds;
+ /// For each operand type, indices of previous/next neighbor instructions with
+ /// defined operands in the instruction list this instruction is a part of.
+ /// \ref NoIdx indicates that there is no such previous or next instruction.
+ std::array<unsigned, NumOprdTypes> Prev, Next;
+
+ ModeInstr() {
+ Oprds.fill(Register());
+ Prev.fill(NoIdx);
+ Next.fill(NoIdx);
+ }
+};
+
+/// Summarizes MSB bits usage over a machine basic block.
+class MBBModeUsage {
+public:
+ /// The machine basic block.
+ const MachineBasicBlock &MBB;
+
+ /// Iterates over \p MBB's instructions to find those for which MSB bits
+ /// provided by MODE are relevant. Indices of virtual registers used at least
+ /// once in an operand reading MSB bits are set in \p OptVirtRegs. Indices of
+ /// those that cannot change MSB group throughout optimization are set in \p
+ /// PinnedVirtRegs.
+ MBBModeUsage(const MachineBasicBlock &MBB, BitVector &OptVirtRegs,
+ BitVector &PinnedVirtRegs);
+
+ /// Returns the list of MODE-using instructions in the block.
+ ArrayRef<ModeInstr> getInstructions() const { return Instructions; }
+
+ /// Returns the MODE-using instruction in the block at index \p Idx.
+ const ModeInstr &getInstruction(unsigned Idx) const {
+ assert(Idx < Instructions.size() && "out of bounds");
+ return Instructions[Idx];
+ }
+
+ /// Returns the index of the first instruction with a MODE-reading operand of
+ /// type \p Oprd in the block, or \ref ModeInstr::NoIdx if none exists.
+ unsigned getFirstOprd(OprdType Oprd) const {
+ return Instructions.empty() ? ModeInstr::NoIdx : getFirstInstrFrom(0, Oprd);
+ }
+
+ /// Returns the index of the last instruction with a MODE-reading operand of
+ /// type \p Oprd in the block, or \ref ModeInstr::NoIdx if none exists.
+ unsigned getLastOprd(OprdType Oprd) const {
+ return Instructions.empty()
+ ? ModeInstr::NoIdx
+ : getLastInstrUntil(Instructions.size() - 1, Oprd);
+ }
+
+ /// Returns the index of the first instruction from \p InstrIdx (included)
+ /// with a MODE-reading operand of type \p Oprd in the block, or \ref
+ /// ModeInstr::NoIdx if none exists.
+ unsigned getFirstInstrFrom(unsigned InstrIdx, OprdType Oprd) const {
+ return getInstrIdxImpl<false, false>(InstrIdx, Oprd);
+ }
+
+ /// Returns the index of the first instruction after \p InstrIdx (excluded)
+ /// with a MODE-reading operand of type \p Oprd in the block, or \ref
+ /// ModeInstr::NoIdx if none exists.
+ unsigned getFirstInstrAfter(unsigned InstrIdx, OprdType Oprd) const {
+ return getInstrIdxImpl<false, true>(InstrIdx, Oprd);
+ }
+
+ /// Returns the index of the last instruction until \p InstrIdx (included)
+ /// with a MODE-reading operand of type \p Oprd in the block, or \ref
+ /// ModeInstr::NoIdx if none exists.
+ unsigned getLastInstrUntil(unsigned InstrIdx, OprdType Oprd) const {
+ return getInstrIdxImpl<true, false>(InstrIdx, Oprd);
+ }
+
+ /// Returns the index of the last instruction before \p InstrIdx (excluded)
+ /// with a MODE-reading operand of type \p Oprd in the block, or \ref
+ /// ModeInstr::NoIdx if none exists.
+ unsigned getLastInstrBefore(unsigned InstrIdx, OprdType Oprd) const {
+ return getInstrIdxImpl<true, true>(InstrIdx, Oprd);
+ }
+
+#ifndef NDEBUG
+ Printable print(const VirtRegMap &VRM) const;
+#endif
+
+private:
+ /// List of all instructions in the machine basic block which have at least
+ /// one operand for which MSB bits provided by MODE are relevant, in program
+ /// order.
+ SmallVector<ModeInstr> Instructions;
+
+ template <bool UsePrev, bool SkipCurrent>
+ unsigned getInstrIdxImpl(unsigned InstrIdx, OprdType Oprd) const;
+};
+
+/// A virtual register whose assigned physical register's MSB group can be
+/// optimized for. An optimizable register maintains a per-group score encoding
+/// its current level of "neighboring-ness" to other registers in each MSB
+/// group, with higher scores indicating a higher number of "neighbor" registers
+/// currently in the corresponding MSB group. Two registers (virtual or
+/// physical) are initially considered neighbors when they are used as operands
+/// of the same type in two neighboring instructions (c.f. \ref ModeInstr). A
+/// register's neighborhood---and thefore its score---changes throughout the
+/// pass's lifetime to reflect the simulataed placement of MODE-setting
+/// instructions.
+///
+/// An optimizable register is said to be "pinned" when the MSB group of its
+/// assigned physical register cannot change. Once a register is pinned it never
+/// becomes unpinned. Score contributions from pinned and unpinned neighbors are
+/// kept separate to enable identification of unoptimizable MSB group conflicts.
+class OptReg {
+public:
+ using WeightedNeighbors = SmallDenseMap<OptReg *, unsigned, 4>;
+
+ /// Abstract coordinates for an occurence of this register.
+ struct Coordinates {
+ /// This index of the MBB.
+ unsigned MBBIndex;
+ /// The index of the MODE-using instruction.
+ unsigned InstrIdx;
+ /// The operand type.
+ OprdType Oprd;
+ };
+
+ /// Creates a neighbor-less optimizable register for register \p VirtReg.
+ /// Neighboring relations with other optimizable registers are
+ /// created/destroyed through class methods.
+ OptReg(Register VirtReg, const VirtRegMap &VRM);
+
+ /// Returns the total number of occurences of pinned neighbor registers in \p
+ /// Group.
+ unsigned getGroupPinnedScore(MSBGroup Group) const {
+ return PinnedScore[Group];
+ }
+
+ /// Returns the total number of occurences of neighbor registers in \p Group.
+ unsigned getGroupScore(MSBGroup Group) const {
+ return PinnedScore[Group] + Score[Group];
+ }
+
+ /// Returns the total number of occurences of neighbor registers in this
+ /// register's current MSB group.
+ unsigned getCurrentGroupScore() const { return getGroupScore(MSB); }
+
+ /// Returns a bitvector the size of the number of MSB groups whose set bits
+ /// indicate the MSB groups in which this register currently has at least one
+ /// pinned neighbor.
+ SmallBitVector getPinGroups() const;
+
+ /// Returns the register's current neighbors.
+ const WeightedNeighbors &getNeighbors() const { return Neighbors; }
+
+ /// Returns the list of coordinates corresponding to this register's
+ /// occurences.
+ ArrayRef<Coordinates> getOccurences() const { return Occurences; }
+
+ /// Returns the underlying virtual register.
+ Register getVirt() const { return VirtReg; }
+
+ /// Returns the underlying virtual register's index.
+ unsigned getVirtIndex() const { return VirtReg.virtRegIndex(); }
+
+ /// Returns the MSB group of this register's currently assigned physical
+ /// register.
+ MSBGroup getMSB() const { return MSB; }
+
+ /// Returns whether the register is pinned.
+ bool isPinned() const { return IsPinned; }
+
+ // addOccurence and record* methods used by OptimizableRegs to initialize the
+ // occurences and neighborhood of all optimizable registers at the beginning.
+
+ /// Adds an occurence of this register in operand type \p Oprd of instruction
+ /// \p InstrIdx of MBB \p MBBIdx.
+ void addOccurence(unsigned MBBIndex, unsigned InstrIdx, OprdType Oprd) {
+ Occurences.push_back({MBBIndex, InstrIdx, Oprd});
+ }
+
+ /// Records an occurence of \p NeighborReg as a neighbor.
+ void recordNeighborOccurence(OptReg &NeighborReg);
+
+ /// Records an occurence of physical register \p PhysReg as a neighbor.
+ void recordPhysNeighborOccurence(Register PhysReg, const VirtRegMap &VRM);
+
+ /// Records an occurence of this register at a block boundary. This adds a
+ /// "pinned occurence" of the default MSB group in which all MBBs start and
+ /// end.
+ void recordBlockBoundaryPin() { ++PinnedScore[DefaultGroup]; }
+
+ // pinMSBGroup and remove* methods used by ModeSetOptimizer to progressively
+ // simplify/destroy the neighborhood of all optimizable registers as it
+ // simulates placement of MODE-setting instructions. remove* methods mirror
+ // record* methods 1-to-1.
+
+ /// Pins this register's to the MSB group of its currently assigned physical
+ /// register.
+ void pinMSBGroup();
+
+ /// Removes an occurence of \p NeighborReg as a neighbor.
+ void removeNeighborOccurence(OptReg &NeighborReg);
+
+ /// Removes an occurence of physical register \p PhysReg as a neighbor.
+ void removePhysNeighborOccurence(Register PhysReg, const VirtRegMap &VRM);
+
+ /// Removes an occurence of this register at a block boundary. This removes a
+ /// "pinned occurence" of the default MSB group in which all MBBs start and
+ /// end.
+ void removeBlockBoundaryPin() {
+ assert(PinnedScore[DefaultGroup] > 0 && "underflow");
+ --PinnedScore[DefaultGroup];
+ }
+
+ /// Notifies the optimizable register that its assigned physcial register has
+ /// changed and that its new assignment belongs to MSB group \p NewGroup. It
+ /// is illegal to change the MSB group of a pinned register.
+ void notifyPhysAssignmentChanged(MSBGroup NewGroup);
+
+#ifndef NDEBUG
+ Printable print(const VirtRegMap &VRM) const;
+#endif
+
+private:
+ /// The virtual register.
+ Register VirtReg;
+ /// MSB group of the virtual register's current physcial register assignment.
+ MSBGroup MSB;
+ /// Per-MSB group score, counting the number of occurences of neighbor
+ /// registers in each group, separated between occurences of unpinned
+ /// optimizable registers from the others (physical registers, pinned
+ /// optimizable registers, and block boundaries). Reflects the register's
+ /// current neighborhood.
+ std::array<unsigned, NumMSBGroups> Score, PinnedScore;
+ /// Maps neighboring optimizable registers to the number of times they occur
+ /// in the neighborhood of one of this register's occurences. Neighbors can be
+ /// added or removed at will after construction, impacting the score.
+ WeightedNeighbors Neighbors;
+ /// Occurences of this register in the function under consideration.
+ /// Occurences can be added after construction but cannot be removed.
+ SmallVector<Coordinates> Occurences;
+ /// Whether the register is pinned to \ref MSB.
+ bool IsPinned = false;
+};
+
+/// Manages all optimizable virtual registers for a function.
+class OptimizableRegs {
+public:
+ /// Creates an \ref OptReg for each virtual register whose index is set in \p
+ /// OptVirtRegs, immediately pinning those whose index is set in \p
+ /// PinnedVirtRegs. Then derives neighborhood of all optimizable registers
+ /// from MODE-using instructions in each block of \p ModeUsage.
+ OptimizableRegs(const BitVector &OptVirtRegs, const BitVector &PinnedVirtRegs,
+ ArrayRef<MBBModeUsage> ModeUsage, const VirtRegMap &VRM);
+
+ OptReg *operator[](Register Reg) {
+ if (Reg.isPhysical())
+ return nullptr;
+ unsigned Idx = VirtRegToStorageIdx[Reg.virtRegIndex()];
+ return Idx == NoIdx ? nullptr : &Storage[Idx];
+ }
+ OptReg &operator[](unsigned VirtRegIdx) {
+ assert(OptVirtRegs.test(VirtRegIdx) && "invalid index");
+ return Storage[VirtRegToStorageIdx[VirtRegIdx]];
+ }
+
+ /// Returns a bitvector whose set bits indicate indices of virtual registers
+ /// which are optimizable i.e., for which (*this)[VirtReg] returns a valid
+ /// \ref OptReg.
+ const BitVector &getAllOptVirtRegs() const { return OptVirtRegs; }
+
+ /// Returns the number of virtual registers, as reported by the MRI.
+ unsigned getNumVirtRegs() const { return OptVirtRegs.size(); }
+
+ using iterator = SmallVector<OptReg>::iterator;
+ using const_iterator = SmallVector<OptReg>::const_iterator;
+ iterator begin() { return Storage.begin(); }
+ iterator end() { return Storage.end(); }
+ const_iterator begin() const { return Storage.begin(); }
+ const_iterator end() const { return Storage.end(); }
+
+private:
+ /// Sentinel value in \p VirtRegToStorageIdx to indicate the non-existence of
+ /// a corresponding \ref OptReg.
+ static constexpr unsigned NoIdx = ~0;
+
+ /// Set bits indicate indices of virtual registers which are optimizable.
+ BitVector OptVirtRegs;
+ /// Backing storage for optimizable registers.
+ SmallVector<OptReg, 0> Storage;
+ /// Works as a map from virtual register indices to the index of the
+ /// corresponding \ref OptReg in \ref Storage. A virtual register that is not
+ /// optimizable "maps to" \ref NoIdx.
+ SmallVector<unsigned, 0> VirtRegToStorageIdx;
+};
+
+/// Simulates placement of MODE-setting instructions as unoptimizable MSB groups
+/// conflicts are detected, driving optimization forward by progressively
+/// pruning register neighborhoods and pinning optimizable registers once they
+/// reach an "ideal" MSB group.
+///
+/// The detection and resolution of unoptimizable MSB group conflicts is this
+/// class's main purpose. An optimizable register with non-null score
+/// contributions from pinned neighbors in more that one MSB group will
+/// necessarily require MODE-setting instructions around its occurences that
+/// neighbor pinned registers in all but one of those MSB groups. This is a
+/// conflict in the sense that we would need the register to be in multiple MSB
+/// groups at the same time to not need MODE-setting instructions. It is
+/// unoptimizable by the pass because pinned neighbors are not allowed to change
+/// MSB group, so no amount of register re-assignment can resolve it. The
+/// objective is to detect those situations early so that no effort is made
+/// attempting to optimize MSB conflits at code locations where we are
+/// guaranteed to be unable to solve them. The class resolves such conflicts by
+/// simulating the placement of MODE-setting instructions around problematic
+/// registers, effectively "breaking" their relationships with some pinned
+/// neighbors until any remaining MSB group conflict becomes optimizable again,
+/// at the known cost of "placed" MODE-setting instructions.
+///
+/// Resolving conflicts strictly lowers the per-group score of optimizable
+/// registers that neighbor "placed" MODE-setting instructions. This ensures
+/// forward progress (scores are lower bounded at 0) and can uncover new
+/// optimization opportunities as register neighborhoods become smaller and some
+/// registers reach an "ideal" MSB group that they can be pinned to.
+///
+/// FIXME: The current approach to determine where we place MODE-setting
+/// instructions to resolve conflicts is correct however when there are multiple
+/// possible locations to choose from it does not attempt to analyze the
+/// expected benefit of each. Picking the best location in such cases should
+/// improve overall pass performance.
+class ModeSetOptimizer {
+public:
+ /// Initializes the optimizer with all optimizable registers in \p OptRegs and
+ /// all blocks in \p ModeUsage. Performs a first round of register pinning and
+ /// conflict resolution on all relevant registers.
+ ModeSetOptimizer(OptimizableRegs &OptRegs, ArrayRef<MBBModeUsage> ModeUsage,
+ const VirtRegMap &VRM);
+
+ /// Iteratively resolves MSB group conflicts by simulating placement of
+ /// MODE-setting instructions and pins newly eligible registers until reaching
+ /// a fixed-point. By the end there are no unoptimizable MSB group conflicts
+ /// and all registers that would be eligible for pinning are pinned. Returns
+ /// whether any register changed state.
+ bool resolveConflictsAndPinRegs();
+
+ /// Notifies the optimizer that \p Reg changed MSB group. Sets bits in \p
+ /// ScoreChanged for all virtual register indices whose score was affected by
+ /// the move. Tracks which registers can become eligible for pinning or may
+ /// exhibit a conflict as a result of the change.
+ void regChangedGroup(OptReg &Reg, BitVector &ScoreChanged);
+
+private:
+ /// Result of BitVector::find* methods when no bit was found.
+ static constexpr int NoBit = -1;
+
+ /// Optimizable registers under consideration.
+ OptimizableRegs &OptRegs;
+ /// Set bits indicate registers which may be pinnable.
+ BitVector CheckShouldBePinned;
+ /// Set bits indicate registers which may have neighbors pinned in more than
+ /// one MSB group.
+ BitVector CheckResolveConflict;
+
+ /// Simulated MODE-setting instruction placement in each machine basic block,
+ /// in the same order as \ref ModeUsage. For any bitvector, a bit at position
+ /// Idx means that there is a mode set placed in between instructions Idx - 1
+ /// (or block entry when Idx == 0) and instruction Idx (or block exit when Idx
+ /// == MBB.Instructions.size()).
+ SmallVector<BitVector> ModeSetPlacement;
+ /// MODE usage in all MBBs.
+ ArrayRef<MBBModeUsage> ModeUsage;
+ const VirtRegMap &VRM;
+
+ /// Resolve conflicts for \p Reg, if any, and returns whether the register had
+ /// conflicts.
+ bool resolveConflictingPins(OptReg &Reg);
+
+ /// Returns whether we consider that \p Reg should be pinned. Registers whose
+ /// only remaining neighbors (pinned or not) are all currently in a single MSB
+ /// group are in the perfect MSB group and should never change group again.
+ bool shouldBePinned(const OptReg &Reg) const;
+
+ /// Pins \p Reg if it is eligible according to \ref shouldbePinned. Returns
+ /// whether the register was newly pinned.
+ bool pinIfEligible(OptReg &Reg);
+
+ /// Determines whether a MODE-setting instruction was already placed in block
+ /// \p MBBIdx between the beginning of the block and MODE-using instruction \p
+ /// InstrIdx.
+ bool hasModeSetBefore(unsigned MBBIdx, unsigned InstrIdx) const {
+ int FirstIdx = ModeSetPlacement[MBBIdx].find_first();
+ return FirstIdx == NoBit ? false
+ : static_cast<unsigned>(FirstIdx) <= InstrIdx;
+ }
+
+ /// Determines whether a MODE-setting instruction was already placed in block
+ /// \p MBBIdx between MODE-using instruction \p InstrIdx and the end of the
+ /// block.
+ bool hasModeSetAfter(unsigned MBBIdx, unsigned InstrIdx) const {
+ int LastIdx = ModeSetPlacement[MBBIdx].find_last();
+ return LastIdx == NoBit ? false : static_cast<unsigned>(LastIdx) > InstrIdx;
+ }
+
+ /// Determines whether a MODE-setting instruction was already placed in block
+ /// \p MBBIdx between MODE-using instructions \p AfterIdx and \p BeforeIdx.
+ bool hasModeSetBetween(unsigned MBBIdx, unsigned AfterIdx,
+ unsigned BeforeIdx) const {
+ assert(AfterIdx < BeforeIdx && "inconsistent indices");
+ // This looks in [AfterIdx + 1, BeforeIdx + 1) == [AfterIdx + 1, BeforeIdx].
+ // This will therefore detect a bit before AfterIdx + 1 (equivalently, after
+ // AfterIdx) and a bit before BeforeIdx.
+ return ModeSetPlacement[MBBIdx].find_first_in(AfterIdx + 1,
+ BeforeIdx + 1) != NoBit;
+ }
+
+ /// Among the MSB groups with non-zero pin score of \p Reg, selects the most
+ /// desirable one in which we would eventually like the register to end up.
+ MSBGroup selectPreferredMSBGroup(const OptReg &Reg) const;
+
+ /// Places a MODE-setting instruction just before MODE-using instruction \p
+ /// InstrIdx in block \p MBBIdx.
+ void placeJustBefore(unsigned MBBIdx, unsigned InstrIdx);
+
+ /// Places a MODE-setting instruction just after MODE-using instruction \p
+ /// InstrIdx in block \p MBBIdx.
+ void placeJustAfter(unsigned MBBIdx, unsigned InstrIdx);
+
+ /// Breaks neighbor relationship between MODE-using instructions \p AfterIdx
+ /// and \p BeforeIdx in block \p MBBIdx and for operand type \p Oprd. Both \p
+ /// AfterIdx and \p BeforeIdx can be \ref ModeInstr::NoIdx in which case they
+ /// encode the default MSB group pin at, respectively, the entry and exit of
+ /// the block.
+ void breakNeighborRelationship(unsigned MBBIdx, unsigned AfterIdx,
+ unsigned BeforeIdx, OprdType Oprd);
+};
+
+/// An optimizable virtual register we consider for re-assignment to a different
+/// MSB group with higher score. Register candidates are weakly ordered,
+/// "larger" candidates being considered more profitable to re-assign to target
+/// MSB groups.
+class OptRegCandidate {
+public:
+ /// Starting epoch when a candidate is created.
+ static constexpr unsigned StartEpoch = 0;
+
+ enum class PinState {
+ /// Single pin in non-target group.
+ PinInBadGroup = 0,
+ /// No pins.
+ NoPin = 1,
+ /// Single pin in target group.
+ TargetIsPin = 2
+ };
+
+ /// The optimizable register.
+ OptReg &Reg;
+
+ /// Set bits indicate MSB groups with the highest score and more desirable
+ /// than the current one the register is in i.e., target groups. No targets
+ /// mean the register is in the best group already.
+ SmallBitVector Targets;
+
+ /// State of pinned neighbors around the register. Only relevant when the
+ /// candidate has at least one target.
+ PinState NeighboringPins;
+
+ /// The overall estimated benefit in re-assigning the register to one of the
+ /// target groups. Higher is better. A negative benefit is still desirable.
+ /// Only relevant when the candidate has at least one target.
+ int Benefit;
+
+ /// Creates the candidate for \p Reg.
+ OptRegCandidate(OptReg &Reg) : Reg(Reg), Targets(NumMSBGroups) {
+ assert(!Reg.isPinned() && "register cannot be pinned initially");
+ update();
+ }
+
+ /// Re-computes the candidate's target groups and potential benefit.
+ void update();
+
+ /// Whether the candidate has a profitable re-assignment to any MSB group
+ /// i.e., a re-assigment that will strictly increase the combined score of all
+ /// optimizable registers.
+ bool isProfitable() const { return Targets.any(); }
+
+ /// Returns the current epoch.
+ unsigned getEpoch() const { return Epoch; }
+
+ /// Bumps the epoch to \p NewEpoch, which must be higher than the current one.
+ void bumpEpoch(unsigned NewEpoch) {
+ assert(Epoch < NewEpoch && "epoch must increase");
+ Epoch = NewEpoch;
+ }
+
+ bool operator<(const OptRegCandidate &Other) const;
+
+#ifndef NDEBUG
+ Printable print(const VirtRegMap &VRM) const;
+#endif
+
+private:
+ /// Last epoch at which a re-assignment took place and during which the
+ /// candidate was evaluated.
+ unsigned Epoch = StartEpoch;
+};
+
+/// A minimal binary max-heap for \ref OptRegCandidate. All tree elements are
+/// created at the beginning and never removed.
+class MaxHeap {
+public:
+ /// Constructs the heap, creating a candidate for each unpinned optimizable
+ /// register tracked by \p OptRegs.
+ MaxHeap(OptimizableRegs &OptRegs);
+
+ /// Returns the most profitable candidate, if any is profitable at all.
+ OptRegCandidate *getMostProfitable() {
+ if (HeapToSlot.empty())
+ return nullptr;
+ OptRegCandidate &TopCand = Slots[HeapToSlot.front()].Cand;
+ return TopCand.isProfitable() ? &TopCand : nullptr;
+ }
+
+ /// If \p Reg is a candidate, update its score and re-sort the heap.
+ void reorderIfExists(const OptReg &Reg);
+
+ /// Returns whether the heap is empty.
+ bool empty() const { return HeapToSlot.empty(); }
+
+#ifndef NDEBUG
+ Printable print(const VirtRegMap &VRM, const LiveIntervals &LIS) const;
+#endif
+
+private:
+ /// A register candidate along with its current heap index.
+ struct Slot {
+ OptRegCandidate Cand;
+ unsigned HeapIdx;
+ Slot(OptReg &Reg, unsigned HeapIdx) : Cand(Reg), HeapIdx(HeapIdx) {}
+ };
+
+ /// Storage for register candidates under consideration.
+ SmallVector<Slot> Slots;
+ /// The heap itself: a complete binary tree, breadth-first, holding indices in
+ /// \ref Slots. The first element is the maximum.
+ SmallVector<unsigned> HeapToSlot;
+ /// Maps registers with a corresponding candidate to the latter's index in
+ /// \ref Slots.
+ DenseMap<const OptReg *, unsigned> OptRegToSlotIdx;
+
+ /// Places slot with index \p SlotIdx at heap position \p HeapIdx, keeping the
+ /// reverse mapping in sync.
+ void place(unsigned HeapIdx, unsigned SlotIdx) {
+ HeapToSlot[HeapIdx] = SlotIdx;
+ Slots[SlotIdx].HeapIdx = HeapIdx;
+ }
+
+ /// Sifts the element at \p HeapIdx towards the root while it outranks its
+ /// parent. Returns true if it moved at all.
+ bool siftUp(unsigned HeapIdx);
+
+ /// Sifts the element at \p HeapIdx towards the leaves while it is outranged
+ /// by its children.
+ bool siftDown(unsigned HeapIdx);
+};
+
+/// Handles re-assignment of virtual registers to physical registers.
+class VirtRegReMap {
+public:
+ VirtRegReMap(LiveRegMatrix &LRM, LiveIntervals &LIS,
+ const MachineFunction &MF, const VirtRegMap &VRM);
+
+ /// Attempts to find an available physical register in MSB group \p Dst that
+ /// virtual register \p Reg can be assigned to. Returns the first such
+ /// register it finds, or the sentinel register if none could be found.
+ MCRegister tryAssignInGroup(Register Reg, MSBGroup Group);
+
+ /// Attempts to re-assign virtual register \p Reg to an available physical
+ /// register in any of the MSB groups indicated by set bits in \p Targets.
+ /// Returns whether any re-assignment took place.
+ bool tryAssignToTargetGroups(OptReg &Reg, const SmallBitVector &Targets);
+
+private:
+ LiveRegMatrix &LRM;
+ LiveIntervals &LIS;
+ RegisterClassInfo RCI;
+ const VirtRegMap &VRM;
+ const SIRegisterInfo &TRI;
+ const BitVector ReservedRegs;
+};
+
+class AMDGPUOptimizeVGPREncoding {
+public:
+ AMDGPUOptimizeVGPREncoding(VirtRegMap &VRM, LiveIntervals &LIS,
+ LiveRegMatrix &LRM)
+ : VRM(VRM), LIS(LIS), LRM(LRM){};
+
+ bool run(MachineFunction &MF);
+
+private:
+ VirtRegMap &VRM;
+ LiveIntervals &LIS;
+ LiveRegMatrix &LRM;
+};
+
+} // namespace
+
+#ifndef NDEBUG
+static std::string printGroup(MSBGroup Group) {
+ return "MSB#" + std::to_string(Group);
+}
+
+static std::string printOprdType(OprdType Oprd) {
+ return "OpTy" + std::to_string(Oprd);
+}
+#endif
+
+/// Returns \p Reg's MSB group. When \p Reg is virtual, \p VRM is used to
+/// determine its current physical assignment.
+static MSBGroup getVGPRGroup(Register Reg, const VirtRegMap &VRM) {
+ const auto &TRI =
+ *static_cast<const SIRegisterInfo *>(&VRM.getTargetRegInfo());
+ MCRegister PhysReg = Reg.isVirtual() ? VRM.getPhys(Reg) : Reg.asMCReg();
+ return TRI.getHWRegIndex(PhysReg) >> 8;
+}
+
+/// Returns true if \p RC is confined to the first 256 VGPRs i.e., every
+/// register it contains has a hardware index below 256 (MSB group 0).
+static bool isLo256VGPRClass(const TargetRegisterClass *RC,
+ const SIRegisterInfo &TRI) {
+ const TargetRegisterClass *Lo256RC =
+ TRI.getAlignedLo256VGPRClassForBitWidth(TRI.getRegSizeInBits(*RC));
+ return (Lo256RC && Lo256RC->hasSubClassEq(RC)) ||
+ Lo256RC == &AMDGPU::VS_32_Lo256RegClass ||
+ Lo256RC == &AMDGPU::VS_64_Lo256RegClass;
+}
+
+MBBModeUsage::MBBModeUsage(const MachineBasicBlock &MBB, BitVector &OptVirtRegs,
+ BitVector &PinnedVirtRegs)
+ : MBB(MBB) {
+
+ const MachineFunction &MF = *MBB.getParent();
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
+ const SIInstrInfo &TII = *ST.getInstrInfo();
+ const SIRegisterInfo &TRI = *ST.getRegisterInfo();
+
+ std::array<unsigned, NumOprdTypes> PrevOprdIndex;
+ PrevOprdIndex.fill(ModeInstr::NoIdx);
+
+ // Identify instructions in the block for which VGPR MSBs are relevant.
+ for (const MachineInstr &MI : MBB) {
+ const MCInstrDesc &Desc = MI.getDesc();
+ const auto [Table, VOPDTable] = AMDGPU::getVGPRLoweringOperandTables(Desc);
+ if (!Table)
+ continue;
+
+ // Determines whether the MI's Name operand has its MSBs provided by MODE.
+ // Returns the underlying machine operand (representing a VGPR) if it is
+ // relevant, otherwise nullptr.
+ auto GetRelevantMO =
+ [&](const AMDGPU::OpName &Name) -> const MachineOperand * {
+ if (Name == AMDGPU::OpName::NUM_OPERAND_NAMES)
+ return nullptr;
+
+ const MachineOperand *MO = TII.getNamedOperand(MI, Name);
+ if (!MO || !MO->isReg())
+ return nullptr;
+
+ Register Reg = MO->getReg();
+ const TargetRegisterClass *RC = TRI.getRegClassForReg(MRI, Reg);
+ return (RC && SIRegisterInfo::isVGPRClass(RC)) ? MO : nullptr;
+ };
+
+ const unsigned InstrIdx = Instructions.size();
+
+ // Sets the register as the operand of a particular type for the current MI.
+ auto SetVGPROprd = [&](Register Reg, OprdType Oprd) -> void {
+ // We need to create the MODE-using instruction if this is the first
+ // MODE-using operand we are seeing for it.
+ if (Instructions.size() == InstrIdx) {
+ ModeInstr &Instr = Instructions.emplace_back();
+ Instr.Prev = PrevOprdIndex;
+ Instr.Oprds[Oprd] = Reg;
+ } else {
+ Instructions.back().Oprds[Oprd] = Reg;
+ }
+
+ // Update next index for all previous instructions that do not use that
+ // operand type. The previous index for this operand type becomes ours for
+ // further instructions.
+ unsigned &PrevOprdIdx = PrevOprdIndex[Oprd];
+ unsigned I = (PrevOprdIdx == ModeInstr::NoIdx) ? 0 : PrevOprdIdx;
+ for (; I < InstrIdx; ++I)
+ Instructions[I].Next[Oprd] = InstrIdx;
+ PrevOprdIdx = InstrIdx;
+ };
+
+ for (OprdType Oprd : seq(NumOprdTypes)) {
+ const MachineOperand *MO = GetRelevantMO(Table[Oprd]);
+ if (!MO)
+ continue;
+ Register Reg = MO->getReg();
+
+ // Tied src2 uses of VOP2 and 32-bit-encoded VOP3 only depend on the vdst
+ // bit and are handled with the def, so they are not their own operand for
+ // MSB group purposes.
+ if (Table[Oprd] == AMDGPU::OpName::src2 && !MO->isDef() && MO->isTied() &&
+ (SIInstrInfo::isVOP2(MI) ||
+ (SIInstrInfo::isVOP3(MI) &&
+ TII.hasVALU32BitEncoding(MI.getOpcode()))))
+ continue;
+
+ SetVGPROprd(Reg, Oprd);
+ if (Reg.isPhysical())
+ continue;
+
+ const unsigned VirtRegIdx = Reg.virtRegIndex();
+ OptVirtRegs.set(VirtRegIdx);
+
+ // For VOPD instructions, we have to take into account that every operand
+ // type would need to fall within the same group in both instructions.
+ // This is outside current modelling capabilities so we just make those
+ // registers pinned by default.
+ if (VOPDTable || isLo256VGPRClass(TRI.getRegClassForReg(MRI, Reg), TRI))
+ PinnedVirtRegs.set(VirtRegIdx);
+ }
+
+ // Pin MODE-using registers for VOPD instructions.
+ if (!VOPDTable)
+ continue;
+ for (OprdType Oprd : seq(NumOprdTypes)) {
+ const MachineOperand *MO = GetRelevantMO(Table[Oprd]);
+ if (!MO)
+ continue;
+ Register Reg = MO->getReg();
+
+ // In case the first operation did not use a MODE-using register for this
+ // operand but the second operation does it is still useful to set the
+ // operand for the current VOPD instruction. This lets other optimizable
+ // registers know there is a pinned register in a particular MSB group at
+ // this location.
+ if (Instructions.size() == InstrIdx || !Instructions.back().Oprds[Oprd])
+ SetVGPROprd(Reg, Oprd);
+
+ if (Reg.isPhysical())
+ continue;
+ const unsigned VirtRegIdx = Reg.virtRegIndex();
+ OptVirtRegs.set(VirtRegIdx);
+ PinnedVirtRegs.set(VirtRegIdx);
+ }
+ }
+}
+
+template <bool UsePrev, bool SkipCurrent>
+unsigned MBBModeUsage::getInstrIdxImpl(unsigned InstrIdx, OprdType Oprd) const {
+ assert(InstrIdx < Instructions.size() && "out of bounds");
+ const ModeInstr &Instr = Instructions[InstrIdx];
+ if constexpr (!SkipCurrent) {
+ if (Instr.Oprds[Oprd])
+ return InstrIdx;
+ }
+
+ unsigned Idx;
+ if constexpr (UsePrev)
+ Idx = Instr.Prev[Oprd];
+ else
+ Idx = Instr.Next[Oprd];
+
+ if (Idx == ModeInstr::NoIdx)
+ return ModeInstr::NoIdx;
+ assert(Instructions[Idx].Oprds[Oprd] && "reg must exist");
+ return Idx;
+}
+
+OptReg::OptReg(Register VirtReg, const VirtRegMap &VRM)
+ : VirtReg(VirtReg), MSB(getVGPRGroup(VirtReg, VRM)) {
+ assert(VirtReg.isVirtual() && "optimizable register should be virtual");
+ PinnedScore.fill(0);
+ Score.fill(0);
+}
+
+SmallBitVector OptReg::getPinGroups() const {
+ SmallBitVector Groups(NumMSBGroups);
+ for (const auto [Group, Score] : enumerate(PinnedScore)) {
+ if (Score != 0)
+ Groups.set(Group);
+ }
+ return Groups;
+}
+
+void OptReg::pinMSBGroup() {
+ if (IsPinned)
+ return;
+
+ IsPinned = true;
+ for (const auto &[NeighborReg, NumOcc] : Neighbors) {
+ assert(NeighborReg->Score[MSB] >= NumOcc);
+ NeighborReg->Score[MSB] -= NumOcc;
+ NeighborReg->PinnedScore[MSB] += NumOcc;
+ }
+}
+
+void OptReg::recordNeighborOccurence(OptReg &NeighborReg) {
+ assert(&NeighborReg != this && "cannot be neighbor with itself");
+ ++Neighbors.insert({&NeighborReg, 0}).first->getSecond();
+ if (NeighborReg.isPinned())
+ ++PinnedScore[NeighborReg.MSB];
+ else
+ ++Score[NeighborReg.MSB];
+}
+
+void OptReg::removeNeighborOccurence(OptReg &NeighborReg) {
+ // Update neighbors.
+ auto Neighbor = Neighbors.find(&NeighborReg);
+ assert(Neighbor != Neighbors.end() && "neighbor must exist");
+ if (--Neighbor->getSecond() == 0)
+ Neighbors.erase(&NeighborReg);
+
+ // Update score.
+ if (NeighborReg.isPinned()) {
+ assert(PinnedScore[NeighborReg.MSB] > 0 && "underflow");
+ --PinnedScore[NeighborReg.MSB];
+ } else {
+ assert(Score[NeighborReg.MSB] > 0 && "underflow");
+ --Score[NeighborReg.MSB];
+ }
+}
+
+void OptReg::recordPhysNeighborOccurence(Register PhysReg,
+ const VirtRegMap &VRM) {
+ assert(PhysReg.isPhysical() && "must be physical register");
+ ++PinnedScore[getVGPRGroup(PhysReg, VRM)];
+}
+
+void OptReg::removePhysNeighborOccurence(Register PhysReg,
+ const VirtRegMap &VRM) {
+ assert(PhysReg.isPhysical() && "must be physical register");
+ MSBGroup PhysGroup = getVGPRGroup(PhysReg, VRM);
+ assert(PinnedScore[PhysGroup] > 0 && "underflow");
+ --PinnedScore[PhysGroup];
+}
+
+void OptReg::notifyPhysAssignmentChanged(MSBGroup NewGroup) {
+ if (MSB == NewGroup)
+ return;
+ assert(!IsPinned && "pinned register cannot change MSB group");
+
+ for (const auto &[NeighborReg, NumOcc] : Neighbors) {
+ assert(NeighborReg->Score[MSB] >= NumOcc);
+ NeighborReg->Score[MSB] -= NumOcc;
+ NeighborReg->Score[NewGroup] += NumOcc;
+ }
+ MSB = NewGroup;
+}
+
+OptimizableRegs::OptimizableRegs(const BitVector &OptVirtRegs,
+ const BitVector &PinnedVirtRegs,
+ ArrayRef<MBBModeUsage> ModeUsage,
+ const VirtRegMap &VRM)
+ : OptVirtRegs(OptVirtRegs), VirtRegToStorageIdx(OptVirtRegs.size(), NoIdx) {
+ assert(OptVirtRegs.size() == PinnedVirtRegs.size() &&
+ "inconsistent bitvector sizes");
+
+ // Create initial tracking data for all optimizable virtual registers.
+ Storage.reserve(OptVirtRegs.size());
+ for (unsigned VirtRegIdx : OptVirtRegs.set_bits()) {
+ VirtRegToStorageIdx[VirtRegIdx] = Storage.size();
+
+ Register VirtReg = Register::index2VirtReg(VirtRegIdx);
+ OptReg &Reg = Storage.emplace_back(VirtReg, VRM);
+ if (PinnedVirtRegs.test(VirtRegIdx))
+ Reg.pinMSBGroup();
+ }
+
+ // Idnetify the neighborhood and occurences of each register. This initializes
+ // the score of all optimizable registers.
+ for (const auto &[MBBIdx, BlockUsage] : enumerate(ModeUsage)) {
+ ArrayRef<ModeInstr> Instructions = BlockUsage.getInstructions();
+ if (Instructions.empty())
+ continue;
+
+ for (OprdType Oprd : seq(NumOprdTypes)) {
+ OptReg *PreviousOptReg = nullptr;
+ unsigned InstrIdx = BlockUsage.getFirstOprd(Oprd);
+
+ while (InstrIdx != ModeInstr::NoIdx) {
+ const ModeInstr &CurrentInstr = Instructions[InstrIdx];
+ Register Reg = CurrentInstr.Oprds[Oprd];
+
+ OptReg *CurrentOptReg = (*this)[Reg];
+ if (CurrentOptReg) {
+ assert(Reg.isVirtual() && "only virtregs are optimizable");
+ CurrentOptReg->addOccurence(MBBIdx, InstrIdx, Oprd);
+
+ if (!PreviousOptReg) {
+ unsigned PrevIdx = CurrentInstr.Prev[Oprd];
+ if (PrevIdx == ModeInstr::NoIdx) {
+ // This is the first operand of that type in the block. Every
+ // block starts with all operand types in the default group. We
+ // model this by incrementing the default's group pinned score for
+ // the first register.
+ CurrentOptReg->recordBlockBoundaryPin();
+ } else {
+ // The register immediately preceeding this was a physical one.
+ Register PhysReg = Instructions[PrevIdx].Oprds[Oprd];
+ CurrentOptReg->recordPhysNeighborOccurence(PhysReg, VRM);
+ }
+ } else if (CurrentOptReg != PreviousOptReg) {
+ // The two virtual registers are neighbors.
+ CurrentOptReg->recordNeighborOccurence(*PreviousOptReg);
+ PreviousOptReg->recordNeighborOccurence(*CurrentOptReg);
+ }
+ } else if (PreviousOptReg) {
+ // We have a virtual register followed by a physical one on the same
+ // operand stream. We just need to update the former's pin score.
+ PreviousOptReg->recordPhysNeighborOccurence(Reg, VRM);
+ PreviousOptReg = nullptr;
+ }
+
+ PreviousOptReg = CurrentOptReg;
+ InstrIdx = CurrentInstr.Next[Oprd];
+ }
+
+ if (PreviousOptReg) {
+ // This is the last operand of that type in the block. Every block
+ // ends with all operand types in the default group. We model this
+ // by incrementing the default's group pinned score for the last
+ // register.
+ PreviousOptReg->recordBlockBoundaryPin();
+ }
+ }
+ }
+}
+
+ModeSetOptimizer::ModeSetOptimizer(OptimizableRegs &OptRegs,
+ ArrayRef<MBBModeUsage> ModeUsage,
+ const VirtRegMap &VRM)
+ : OptRegs(OptRegs), CheckShouldBePinned(OptRegs.getAllOptVirtRegs()),
+ CheckResolveConflict(OptRegs.getNumVirtRegs()),
+ ModeSetPlacement(ModeUsage.size()), ModeUsage(ModeUsage), VRM(VRM) {
+ for (const auto &[MBB, Placement] : zip_equal(ModeUsage, ModeSetPlacement))
+ Placement.resize(MBB.getInstructions().size() + 1);
+
+ // We initially check all registers for pin-eligibility, and those that are
+ // neighbors of already pinned registers for conflicting pins.
+ for (const OptReg &Reg : OptRegs) {
+ if (!Reg.isPinned())
+ continue;
+ for (const auto &[NeighborReg, _] : Reg.getNeighbors())
+ CheckResolveConflict.set(NeighborReg->getVirtIndex());
+ }
+ resolveConflictsAndPinRegs();
+}
+
+MSBGroup ModeSetOptimizer::selectPreferredMSBGroup(const OptReg &Reg) const {
+ // Initially favor the current group the register is in. This is guaranteed to
+ // change if the register has no pinned neighbors in this MSB group.
+ MSBGroup BestGroup = Reg.getMSB();
+ unsigned MaxPinnedScore = Reg.getGroupPinnedScore(BestGroup),
+ MaxTotalScore = Reg.getGroupScore(BestGroup);
+
+ for (MSBGroup Group : seq(NumMSBGroups)) {
+ if (Group == Reg.getMSB())
+ continue;
+
+ // Select the MSB group with the highest number of pinned neighbors.
+ unsigned PinnedScore = Reg.getGroupPinnedScore(Group);
+ if (PinnedScore < MaxPinnedScore)
+ continue;
+
+ // Ammong MSB groups with the same number of pinned neighbors, favor the one
+ // with highest number of unpinned neighbors.
+ unsigned TotalScore = Reg.getGroupScore(Group);
+ if (PinnedScore == MaxPinnedScore && MaxTotalScore > TotalScore)
+ continue;
+
+ BestGroup = Group;
+ MaxPinnedScore = PinnedScore;
+ MaxTotalScore = TotalScore;
+ }
+ return BestGroup;
+}
+
+void ModeSetOptimizer::placeJustBefore(unsigned MBBIdx, unsigned InstrIdx) {
+ const MBBModeUsage &MBB = ModeUsage[MBBIdx];
+ for (OprdType Oprd : seq(NumOprdTypes)) {
+ breakNeighborRelationship(MBBIdx, MBB.getLastInstrBefore(InstrIdx, Oprd),
+ MBB.getFirstInstrFrom(InstrIdx, Oprd), Oprd);
+ }
+ ModeSetPlacement[MBBIdx].set(InstrIdx);
+}
+
+void ModeSetOptimizer::placeJustAfter(unsigned MBBIdx, unsigned InstrIdx) {
+ const MBBModeUsage &MBB = ModeUsage[MBBIdx];
+ for (OprdType Oprd : seq(NumOprdTypes)) {
+ breakNeighborRelationship(MBBIdx, MBB.getLastInstrUntil(InstrIdx, Oprd),
+ MBB.getFirstInstrAfter(InstrIdx, Oprd), Oprd);
+ }
+ ModeSetPlacement[MBBIdx].set(InstrIdx + 1);
+}
+
+void ModeSetOptimizer::breakNeighborRelationship(unsigned MBBIdx,
+ unsigned AfterIdx,
+ unsigned BeforeIdx,
+ OprdType Oprd) {
+ if (AfterIdx == ModeInstr::NoIdx && BeforeIdx == ModeInstr::NoIdx)
+ return;
+
+ ArrayRef<ModeInstr> Instructions = ModeUsage[MBBIdx].getInstructions();
+ if (AfterIdx == ModeInstr::NoIdx) {
+ // This may break the relationship between an optimizable register and the
+ // entry block pin.
+ Register BeforeReg = Instructions[BeforeIdx].Oprds[Oprd];
+ if (OptRegs[BeforeReg] && !hasModeSetBefore(MBBIdx, BeforeIdx))
+ OptRegs[BeforeReg]->removeBlockBoundaryPin();
+ return;
+ }
+ if (BeforeIdx == ModeInstr::NoIdx) {
+ // This may break the relationship between an optimizable register and the
+ // exit block pin.
+ Register AfterReg = Instructions[AfterIdx].Oprds[Oprd];
+ if (OptRegs[AfterReg] && !hasModeSetAfter(MBBIdx, AfterIdx))
+ OptRegs[AfterReg]->removeBlockBoundaryPin();
+ return;
+ }
+ assert(AfterIdx < BeforeIdx && "incoherent indices");
+
+ // The two neighbor registers on that operand lane are already not neighbors
+ // if there is a MODE-setting instruction between them.
+ if (hasModeSetBetween(MBBIdx, AfterIdx, BeforeIdx))
+ return;
+
+ // A register is never considered a neighbor to itself.
+ Register AfterReg = Instructions[AfterIdx].Oprds[Oprd];
+ Register BeforeReg = Instructions[BeforeIdx].Oprds[Oprd];
+ assert(AfterReg && BeforeReg && "register operands must exist");
+ if (AfterReg == BeforeReg)
+ return;
+
+ // Notify registers that one occurence of their neighborhood relationship is
+ // broken. Optimizable registers which have their score affected by the
+ // break may become pinnable.
+ OptReg *AferOptReg = OptRegs[AfterReg];
+ OptReg *BeforeOptReg = OptRegs[BeforeReg];
+ if (AferOptReg && BeforeOptReg) {
+ AferOptReg->removeNeighborOccurence(*BeforeOptReg);
+ BeforeOptReg->removeNeighborOccurence(*AferOptReg);
+ CheckShouldBePinned.set(AfterReg.virtRegIndex());
+ CheckShouldBePinned.set(BeforeReg.virtRegIndex());
+ } else if (AferOptReg) {
+ AferOptReg->removePhysNeighborOccurence(BeforeReg, VRM);
+ CheckShouldBePinned.set(AfterReg.virtRegIndex());
+ } else if (BeforeOptReg) {
+ BeforeOptReg->removePhysNeighborOccurence(AfterReg, VRM);
+ CheckShouldBePinned.set(BeforeReg.virtRegIndex());
+ }
+}
+
+bool ModeSetOptimizer::resolveConflictingPins(OptReg &Reg) {
+ if (Reg.getPinGroups().count() <= 1)
+ return false;
+
+ LLVM_DEBUG(dbgs() << " Resolving conflicts for " << Reg.print(VRM) << '\n');
+
+ // Only one MSB group with pinned neighbors must remain.
+ MSBGroup PreferredGroup = selectPreferredMSBGroup(Reg);
+ assert(PreferredGroup < NumMSBGroups && "invalid group");
+
+ LLVM_DEBUG(dbgs() << " Preferred group is " << printGroup(PreferredGroup)
+ << '\n');
+
+ // Conflicts with block boundary pins may need to be resolved when the default
+ // group is not the preferred one.
+ if (PreferredGroup != DefaultGroup &&
+ Reg.getGroupPinnedScore(DefaultGroup) > 0) {
+ for (const auto &[MBBIdx, InstrIdx, Oprd] : Reg.getOccurences()) {
+ const MBBModeUsage &MBB = ModeUsage[MBBIdx];
+ if (InstrIdx == MBB.getFirstOprd(Oprd) &&
+ !hasModeSetBefore(MBBIdx, InstrIdx)) {
+ LLVM_DEBUG(dbgs() << " Resolving default entry pin "
+ << printOprdType(Oprd) << " in MBB#"
+ << MBB.MBB.getNumber() << '\n');
+ placeJustBefore(MBBIdx, InstrIdx);
+ }
+ if (InstrIdx == MBB.getLastOprd(Oprd) &&
+ !hasModeSetAfter(MBBIdx, InstrIdx)) {
+ LLVM_DEBUG(dbgs() << " Resolving default exit pin "
+ << printOprdType(Oprd) << " in MBB#"
+ << MBB.MBB.getNumber() << '\n');
+ placeJustAfter(MBBIdx, InstrIdx);
+ }
+ }
+ }
+
+ // Conflicts with occurences of pinned neighbors in the non-preferred MSB
+ // group need to be resolved. Iterate over a copy of the list of neighbors
+ // because they will be modified as we simulate placement of MODE-setting
+ // instructions.
+ OptReg::WeightedNeighbors Neighbors(Reg.getNeighbors());
+ for (const auto &[NeighborReg, _] : Neighbors) {
+ // Early exit when we know we are not gonna find conflicting occurences.
+ if (!NeighborReg->isPinned() || NeighborReg->getMSB() == PreferredGroup ||
+ Reg.getGroupPinnedScore(NeighborReg->getMSB()) == 0)
+ continue;
+
+ // Look through occurences for conflicts.
+ for (const auto &[MBBIdx, InstrIdx, Oprd] : NeighborReg->getOccurences()) {
+ const MBBModeUsage &MBB = ModeUsage[MBBIdx];
+ ArrayRef<ModeInstr> Instructions = MBB.getInstructions();
+
+ // Look at the operand immediately before this neighbor occurence. If it
+ // matches the register for which we are currently resolving conflicts,
+ // then it is one of the neighborhood relationship to break.
+ unsigned PrevIdx = MBB.getLastInstrBefore(InstrIdx, Oprd);
+ if (PrevIdx != ModeInstr::NoIdx &&
+ Instructions[PrevIdx].Oprds[Oprd] == Reg.getVirt() &&
+ !hasModeSetBetween(MBBIdx, PrevIdx, InstrIdx)) {
+
+ LLVM_DEBUG(dbgs() << " Resolving with next neighbor "
+ << NeighborReg->print(VRM) << " for operand type "
+ << printOprdType(Oprd) << " in MBB#"
+ << MBB.MBB.getNumber() << '\n');
+ placeJustBefore(MBBIdx, InstrIdx);
+ }
+
+ // Look at the operand immediately after this neighbor occurence. If it
+ // matches the register for which we are currently resolving conflicts,
+ // then it is one of the neighborhood relationship to break.
+ unsigned NextIdx = MBB.getFirstInstrAfter(InstrIdx, Oprd);
+ if (NextIdx != ModeInstr::NoIdx &&
+ Instructions[NextIdx].Oprds[Oprd] == Reg.getVirt() &&
+ !hasModeSetBetween(MBBIdx, InstrIdx, NextIdx)) {
+ LLVM_DEBUG(dbgs() << " Resolving with previous neighbor "
+ << NeighborReg->print(VRM) << " for operand type "
+ << printOprdType(Oprd) << " in MBB#"
+ << MBB.MBB.getNumber() << '\n');
+ placeJustBefore(MBBIdx, NextIdx);
+ }
+ }
+ }
+
+ LLVM_DEBUG(dbgs() << " | Updated register: " << Reg.print(VRM) << '\n');
+
+ // It is possible that the number of pinned groups was reduced to zero when
+ // all of the preferred group's pins were close to other groups' respective
+ // pins.
+ assert(Reg.getPinGroups().count() <= 1 && "at most one pin");
+ return true;
+}
+
+bool ModeSetOptimizer::shouldBePinned(const OptReg &Reg) const {
+ for (MSBGroup Group : seq(NumMSBGroups)) {
+ if (Reg.getGroupScore(Group) == 0)
+ continue;
+ if (Group != Reg.getMSB())
+ return false;
+ }
+ return true;
+}
+
+bool ModeSetOptimizer::pinIfEligible(OptReg &Reg) {
+ if (Reg.isPinned() || !shouldBePinned(Reg))
+ return false;
+ Reg.pinMSBGroup();
+ LLVM_DEBUG(dbgs() << " Pinned " << Reg.print(VRM) << " to "
+ << printGroup(Reg.getMSB()) << '\n');
+ return true;
+}
+
+void ModeSetOptimizer::regChangedGroup(OptReg &Reg, BitVector &ScoreChanged) {
+ // The candidate register and its neighbors had their score changed; they may
+ // also be eligible for pinning.
+ pinIfEligible(Reg);
+ ScoreChanged.set(Reg.getVirtIndex());
+
+ for (const auto &[NeighborReg, _] : Reg.getNeighbors()) {
+ ScoreChanged.set(NeighborReg->getVirtIndex());
+ if (!pinIfEligible(*NeighborReg))
+ continue;
+ // Pinning a register impacts its neighbors' respective score. However, they
+ // cannot become newly pinnable themselves because pin-eligibility is
+ // independent of whether score contributions come from pinned or unpinned
+ // neighbors.
+ for (const auto &[SecondDegNeighbor, _] : NeighborReg->getNeighbors())
+ ScoreChanged.set(SecondDegNeighbor->getVirtIndex());
+ }
+
+ // A score change may mean the register now has a conflict.
+ CheckResolveConflict |= ScoreChanged;
+}
+
+bool ModeSetOptimizer::resolveConflictsAndPinRegs() {
+ LLVM_DEBUG(dbgs() << "* Resolving conflicts and pins:\n");
+ bool AnyChange = false;
+ do {
+ // Conflict resolution.
+ for (unsigned VirtRegIdx : CheckResolveConflict.set_bits())
+ AnyChange |= resolveConflictingPins(OptRegs[VirtRegIdx]);
+ CheckResolveConflict.reset();
+
+ // Pin eligible registers.
+ for (unsigned VirtRegIdx : CheckShouldBePinned.set_bits()) {
+ OptReg &Reg = OptRegs[VirtRegIdx];
+ if (!pinIfEligible(Reg))
+ continue;
+ AnyChange = true;
+
+ // Neighbors of registers which became pinned may now have conflicts.
+ for (const auto &[NeighborReg, _] : Reg.getNeighbors())
+ CheckResolveConflict.set(NeighborReg->getVirtIndex());
+ }
+ CheckShouldBePinned.reset();
+ } while (CheckResolveConflict.any());
+ return AnyChange;
+}
+
+bool OptRegCandidate::operator<(const OptRegCandidate &Other) const {
+ // Registers without targets are not useful candidates. Pinned registers have
+ // no targets by construction so they get caught here as well.
+ if (Other.Targets.none())
+ return Targets.none();
+ if (Targets.none())
+ return true;
+
+ // Earlier/Smaller epoch wins.
+ if (Epoch != Other.Epoch)
+ return Other.Epoch < Epoch;
+
+ // Bigger pin-state wins. This generally favors MSB group changes toward
+ // groups with bigger number of pinned neighbors in.
+ if (NeighboringPins != Other.NeighboringPins)
+ return NeighboringPins < Other.NeighboringPins;
+
+ // Higher benefit wins.
+ if (Benefit < Other.Benefit)
+ return Benefit < Other.Benefit;
+
+ // Break ties with unique virtual register index.
+ return Reg.getVirtIndex() < Other.Reg.getVirtIndex();
+}
+
+void OptRegCandidate::update() {
+ Targets.reset();
+ if (Reg.isPinned())
+ return;
+
+ SmallBitVector PinGroups = Reg.getPinGroups();
+ if (PinGroups.count() > 1) {
+ // Registers with unresolved conflicts should not be re-assigned until we
+ // have resolved them.
+ return;
+ }
+ const unsigned CurrentScore = Reg.getCurrentGroupScore();
+
+ // The overall benefit of moving MSB group reflects the distribution of scores
+ // over all MSB groups. We want to favor registers with highly unbalanced
+ // per-group score, at the extreme registers whose neighborhood is entirely
+ // assigned to a single MSB group. On the contrary, registers whose
+ // neighborhood is roughly evenly split between all MSB groups are not that
+ // profitable to re-assign; even if a MSB group with higher score than the
+ // current one the register is in exists.
+ Benefit = CurrentScore;
+ unsigned MaxGroupScore = CurrentScore;
+ for (MSBGroup Group : seq(NumMSBGroups)) {
+ if (Group == Reg.getMSB())
+ continue;
+ const unsigned GroupScore = Reg.getGroupScore(Group);
+ if (GroupScore > MaxGroupScore) {
+ // The current best groups' score was counted positively once before, but
+ // we now want to count it negatively.
+ Benefit -= MaxGroupScore * 2;
+ // The new highest group score is counted positively.
+ Benefit += GroupScore;
+ // This group becomes the only target.
+ Targets.reset();
+ Targets.set(Group);
+ MaxGroupScore = GroupScore;
+ continue;
+ }
+ // GroupScore <= MaxGroupScore
+
+ Benefit -= GroupScore;
+ // This group is a good target only when it has the highest score we have
+ // seen so far and that score is above the current group's score.
+ if (GroupScore != CurrentScore && GroupScore == MaxGroupScore)
+ Targets.set(Group);
+ }
+ assert(!Targets.test(Reg.getMSB()) && "current group cannot be target");
+ if (Targets.none())
+ return;
+
+ // We want to favor registers which would benefit from being re-assigned to
+ // the single MSB group in which they haved pinned neighbors (if there is
+ // one), as this is likely to resolve conflicts in the future without having
+ // to place MODE-setting instructions.
+ if (PinGroups.none()) {
+ NeighboringPins = PinState::NoPin;
+ } else {
+ const MSBGroup SinglePin = PinGroups.find_first();
+ NeighboringPins = (SinglePin == Reg.getMSB() || !Targets.test(SinglePin))
+ ? PinState::PinInBadGroup
+ : PinState::TargetIsPin;
+ }
+}
+
+MaxHeap::MaxHeap(OptimizableRegs &OptRegs) {
+ for (OptReg &Reg : OptRegs) {
+ if (Reg.isPinned())
+ continue;
+
+ const unsigned SlotIdx = Slots.size();
+ const unsigned HeapIdx = HeapToSlot.size();
+ Slots.emplace_back(Reg, HeapIdx);
+ HeapToSlot.push_back(SlotIdx);
+ OptRegToSlotIdx.insert({&Reg, SlotIdx});
+ siftUp(HeapIdx);
+ }
+}
+
+void MaxHeap::reorderIfExists(const OptReg &Reg) {
+ auto Cand = OptRegToSlotIdx.find(&Reg);
+ if (Cand == OptRegToSlotIdx.end())
+ return;
+ Slot &S = Slots[Cand->second];
+ S.Cand.update();
+
+ // Re-order the tree around the updated slot.
+ if (!siftUp(S.HeapIdx))
+ siftDown(S.HeapIdx);
+}
+
+bool MaxHeap::siftUp(unsigned HeapIdx) {
+ unsigned S = HeapToSlot[HeapIdx];
+ bool Moved = false;
+ while (HeapIdx != 0) {
+ unsigned Parent = (HeapIdx - 1) / 2;
+ if (!(Slots[HeapToSlot[Parent]].Cand < Slots[S].Cand))
+ break;
+ place(HeapIdx, HeapToSlot[Parent]);
+ HeapIdx = Parent;
+ Moved = true;
+ }
+ if (Moved)
+ place(HeapIdx, S);
+ return Moved;
+}
+
+bool MaxHeap::siftDown(unsigned HeapIdx) {
+ unsigned S = HeapToSlot[HeapIdx];
+ unsigned N = HeapToSlot.size();
+ bool Moved = false;
+ while (true) {
+ unsigned Left = 2 * HeapIdx + 1;
+ unsigned Right = Left + 1;
+ unsigned Largest = HeapIdx;
+ if (Left < N && Slots[S].Cand < Slots[HeapToSlot[Left]].Cand)
+ Largest = Left;
+ if (Right < N &&
+ Slots[HeapToSlot[Largest]].Cand < Slots[HeapToSlot[Right]].Cand)
+ Largest = Right;
+ if (Largest == HeapIdx)
+ break;
+ place(HeapIdx, HeapToSlot[Largest]);
+ HeapIdx = Largest;
+ Moved = true;
+ }
+ if (Moved)
+ place(HeapIdx, S);
+ return Moved;
+}
+
+VirtRegReMap::VirtRegReMap(LiveRegMatrix &LRM, LiveIntervals &LIS,
+ const MachineFunction &MF, const VirtRegMap &VRM)
+ : LRM(LRM), LIS(LIS), VRM(VRM),
+ TRI(*static_cast<const SIRegisterInfo *>(&VRM.getTargetRegInfo())),
+ ReservedRegs(TRI.getReservedRegs(MF)) {
+ RCI.runOnMachineFunction(MF);
+}
+
+static bool overflowsLastGroup(unsigned HWRegIdx, unsigned NumLanes,
+ MSBGroup Group) {
+ if (Group != NumMSBGroups - 1)
+ return false;
+ return ((HWRegIdx + NumLanes - 1) >> 8) != NumMSBGroups - 1;
+}
+
+MCRegister VirtRegReMap::tryAssignInGroup(Register VirtReg, MSBGroup Group) {
+ assert(VirtReg.isVirtual() && "expected virtreg");
+ const LiveInterval &RegLI = LIS.getInterval(VirtReg);
+ const MachineRegisterInfo &MRI = VRM.getRegInfo();
+ const TargetRegisterClass &RC = *MRI.getRegClass(VirtReg);
+ const unsigned NumLanes = divideCeil(TRI.getRegSizeInBits(RC), 32);
+ for (MCPhysReg CandPhysReg : RCI.getOrder(&RC)) {
+ if (ReservedRegs[CandPhysReg] || getVGPRGroup(CandPhysReg, VRM) != Group ||
+ overflowsLastGroup(TRI.getHWRegIndex(CandPhysReg), NumLanes, Group))
+ continue;
+ if (LRM.checkInterference(RegLI, CandPhysReg) == LiveRegMatrix::IK_Free)
+ return CandPhysReg;
+ }
+ return MCPhysReg();
+}
+
+bool VirtRegReMap::tryAssignToTargetGroups(OptReg &Reg,
+ const SmallBitVector &Targets) {
+ Register VirtReg = Reg.getVirt();
+ const LiveInterval &LI = LIS.getInterval(VirtReg);
+
+ MCRegister OriginalPhys = VRM.getPhys(VirtReg);
+ LRM.unassign(LI);
+
+ for (MSBGroup Target : Targets.set_bits()) {
+ assert(Target != Reg.getMSB() && "target is current MSB group");
+ MCRegister NewPhysReg = tryAssignInGroup(VirtReg, Target);
+ if (NewPhysReg) {
+ LRM.assign(LI, NewPhysReg);
+ Reg.notifyPhysAssignmentChanged(Target);
+ return true;
+ }
+ }
+ // We failed to find a register, re-assign the original one.
+ LRM.assign(LI, OriginalPhys);
+ return false;
+}
+
+bool AMDGPUOptimizeVGPREncoding::run(MachineFunction &MF) {
+ const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
+ if (!ST.has1024AddressableVGPRs())
+ return false;
+
+ LLVM_DEBUG(dbgs() << "*** AMDGPUOptimizeVGPREncoding on " << MF.getName()
+ << " ***\n");
+ const MachineRegisterInfo &MRI = VRM.getRegInfo();
+
+ // Analyze MODE-usage in each block, keeping track of which virtual registers
+ // are used in relevant instruction operands.
+ BitVector OptVirtRegs(MRI.getNumVirtRegs()), Pinned(MRI.getNumVirtRegs());
+ SmallVector<MBBModeUsage> ModeUsage;
+ ModeUsage.reserve(MF.getNumBlockIDs());
+ for (const MachineBasicBlock &MBB : MF)
+ ModeUsage.emplace_back(MBB, OptVirtRegs, Pinned);
+
+ // Initialize all optimizable registers.
+ OptimizableRegs OptRegs(OptVirtRegs, Pinned, ModeUsage, VRM);
+
+ LLVM_DEBUG({
+ dbgs() << "* Per-block MODE-usage:\n";
+ for (const MBBModeUsage &MBB : ModeUsage) {
+ dbgs() << " MBB #" << MBB.MBB.getNumber() << ":\n" << MBB.print(VRM);
+ }
+ dbgs() << "* Neighborhoods:\n";
+ for (const OptReg &Reg : OptRegs) {
+ dbgs() << " " << Reg.print(VRM) << '\n';
+ for (const auto &[NeighborReg, NumOcc] : Reg.getNeighbors()) {
+ dbgs() << " [" << NumOcc << "] " << NeighborReg->print(VRM) << '\n';
+ }
+ }
+ });
+
+ ModeSetOptimizer Optimizer(OptRegs, ModeUsage, VRM);
+ VirtRegReMap VRRM(LRM, LIS, MF, VRM);
+
+ bool Changed = false;
+ do {
+ // Constructs a max-heap with all remaining unpinned registers that are
+ // candidates for re-assignment.
+ //
+ /// FIXME: There is no need to re-construct the heap every time, we can just
+ /// let pin registers fall to the bottom of it since they have no target MSB
+ /// group by construction.
+ MaxHeap AllCandidates(OptRegs);
+ LLVM_DEBUG(dbgs() << AllCandidates.print(VRM, LIS));
+
+ // Incremented at each successful register re-assignment, which may unblock
+ // previously failed attempts at re-assigning.
+ unsigned Epoch = OptRegCandidate::StartEpoch + 1;
+
+ // Keeps track of which optimizable virtual registers have their score
+ // changed by re-assignments.
+ BitVector ScoreChanged(OptRegs.getNumVirtRegs());
+
+ // Only accepting profitable candidates guarantees forward progress
+ // because re-assigning to target groups increase the combined score, which
+ // is upper-bounded by the number of neighboring relationships between all
+ // registers (itself only decreasing after initialization).
+ while (OptRegCandidate *Candidate = AllCandidates.getMostProfitable()) {
+ LLVM_DEBUG({
+ dbgs() << "| Attempting re-assignment of " << Candidate->print(VRM)
+ << '\n';
+ for (const auto &[NeighborReg, _] : Candidate->Reg.getNeighbors())
+ dbgs() << " " << NeighborReg->print(VRM) << '\n';
+ });
+
+ // The epoch check catches a second evaluation of the same candidate under
+ // the same exact conditions, which would fail again.
+ if (Candidate->getEpoch() == Epoch) {
+ LLVM_DEBUG(dbgs() << " | No more useful candidates!\n");
+ break;
+ }
+ Candidate->bumpEpoch(Epoch);
+ ScoreChanged.reset();
+
+ // The candidate's epoch changed.
+ ScoreChanged.set(Candidate->Reg.getVirtIndex());
+
+ // Attempts re-assignment to a better MSB group.
+ if (VRRM.tryAssignToTargetGroups(Candidate->Reg, Candidate->Targets)) {
+ LLVM_DEBUG(dbgs() << " | SUCCESS: Assigned to free physical register "
+ << VRM.getPhys(Candidate->Reg.getVirt()) << '\n');
+ ++Epoch;
+ Optimizer.regChangedGroup(Candidate->Reg, ScoreChanged);
+ }
+
+ // Optimizable registers whose score changed need to be re-ordered within
+ // the max-heap.
+ for (unsigned VirtRegIdx : ScoreChanged.set_bits())
+ AllCandidates.reorderIfExists(OptRegs[VirtRegIdx]);
+ }
+ Changed |= (Epoch != OptRegCandidate::StartEpoch + 1);
+ } while (Optimizer.resolveConflictsAndPinRegs());
+
+ return Changed;
+}
+
+#ifndef NDEBUG
+
+Printable MBBModeUsage::print(const VirtRegMap &VRM) const {
+ // 16 characters is enough for a virtual register with 9 digits, or to
+ // display the beginning of a wide physical register.
+ static constexpr unsigned ColumnWidth = 16;
+
+ return Printable([&](raw_ostream &OS) {
+ const MachineRegisterInfo &MRI = VRM.getRegInfo();
+
+ std::array<std::optional<MSBGroup>, NumOprdTypes> GroupPerOprd;
+ GroupPerOprd.fill(0);
+
+ // Display operand type names.
+ OS << " [ ";
+ for (OprdType Oprd : seq(NumOprdTypes)) {
+ SmallString<ColumnWidth> Buf;
+ raw_svector_ostream OprdStream(Buf);
+ OprdStream << printOprdType(Oprd);
+ StringRef OprdStr = OprdStream.str();
+ OprdStr = OprdStr.take_front(ColumnWidth - 2);
+ OS << OprdStr << indent(ColumnWidth - OprdStr.size());
+ }
+ OS << " ]\n";
+
+ for (const ModeInstr &Instr : Instructions) {
+ OS << " [ ";
+ for (const auto &[Oprd, Reg] : enumerate(Instr.Oprds)) {
+ if (!Reg) {
+ OS << '|' << indent(ColumnWidth - 1);
+ // No operand of that type for that instruction means no MSB
+ // requirement.
+ continue;
+ }
+
+ SmallString<ColumnWidth> Buf;
+ raw_svector_ostream RegStream(Buf);
+ RegStream << printReg(Reg, &VRM.getTargetRegInfo(), 0, &MRI);
+ StringRef RegStr = RegStream.str();
+ RegStr = RegStr.take_front(ColumnWidth);
+ OS << RegStr << indent(ColumnWidth - RegStr.size());
+
+ MSBGroup OprdGroup = getVGPRGroup(Reg, VRM);
+ std::optional<MSBGroup> &CurrentOprdGroup = GroupPerOprd[Oprd];
+ if (CurrentOprdGroup.has_value()) {
+ if (*CurrentOprdGroup != OprdGroup) {
+ // Different group as previously used for that operand.
+ CurrentOprdGroup = OprdGroup;
+ }
+ } else {
+ // First operand in the sequence. This defines the initial desired
+ // group for that operand in the instruction sequence, so we do not
+ // account for the set MSB here.
+ CurrentOprdGroup = OprdGroup;
+ }
+ }
+ OS << " ]\n";
+ }
+ });
+}
+
+Printable OptReg::print(const VirtRegMap &VRM) const {
+ return Printable([&](raw_ostream &OS) {
+ const auto &TRI =
+ *static_cast<const SIRegisterInfo *>(&VRM.getTargetRegInfo());
+ const MachineRegisterInfo &MRI = VRM.getRegInfo();
+ OS << printReg(VirtReg, &TRI, 0, &MRI) << '/'
+ << printReg(VRM.getPhys(VirtReg), &TRI, 0, &MRI) << '/'
+ << printGroup(getVGPRGroup(VirtReg, VRM));
+ if (isPinned())
+ OS << " (pinned)";
+ OS << " [";
+ for (MSBGroup Group : seq(NumMSBGroups)) {
+ OS << (Group == MSB ? '+' : '-') << Score[Group] << '('
+ << PinnedScore[Group] << ')';
+ if (Group != NumMSBGroups - 1)
+ OS << ", ";
+ }
+ OS << ']';
+ });
+}
+
+Printable OptRegCandidate::print(const VirtRegMap &VRM) const {
+ return Printable([&](raw_ostream &OS) {
+ OS << Reg.print(VRM);
+ if (Targets.none()) {
+ OS << " has no targets";
+ return;
+ }
+
+ auto PrintMoveScore = [&](MSBGroup MSB) {
+ OS << printGroup(MSB) << "(+"
+ << Reg.getGroupScore(MSB) - Reg.getCurrentGroupScore() << ')';
+ };
+
+ OS << " move to ";
+ for (MSBGroup Dst : drop_end(Targets.set_bits())) {
+ PrintMoveScore(Dst);
+ OS << " / ";
+ }
+ PrintMoveScore(Targets.find_last());
+ OS << " with benefit " << Benefit;
+
+ OS << " (";
+ switch (NeighboringPins) {
+ case PinState::PinInBadGroup:
+ OS << "bad pin";
+ break;
+ case PinState::NoPin:
+ OS << "no pin";
+ break;
+ case PinState::TargetIsPin:
+ OS << "move to pin";
+ break;
+ }
+ OS << ") @ " << "epoch " << Epoch;
+ });
+}
+
+Printable MaxHeap::print(const VirtRegMap &VRM,
+ const LiveIntervals &LIS) const {
+ return Printable([&](raw_ostream &OS) {
+ auto PrintRegsList = [&](ArrayRef<const OptRegCandidate *> Cands,
+ StringRef Name) {
+ OS << " " << Name << " (" << Cands.size() << " registers):\n";
+ for (const OptRegCandidate *Cand : Cands)
+ OS << " " << Cand->print(VRM) << '\n';
+ };
+
+ auto PrintRegs = [&](ArrayRef<const OptRegCandidate *> Cands) -> void {
+ using PinState = OptRegCandidate::PinState;
+ // Group candidates per pin state.
+ SmallDenseMap<PinState, SmallVector<const OptRegCandidate *>, 4>
+ CandsPerPinState;
+ SmallVector<const OptRegCandidate *> NoTargets;
+ for (const OptRegCandidate *Cand : Cands) {
+ if (Cand->Targets.any())
+ CandsPerPinState[Cand->NeighboringPins].push_back(Cand);
+ else
+ NoTargets.push_back(Cand);
+ }
+ PrintRegsList(CandsPerPinState[PinState::TargetIsPin], "Good pin");
+ PrintRegsList(CandsPerPinState[PinState::NoPin], "No pin");
+ PrintRegsList(CandsPerPinState[PinState::PinInBadGroup], "Bad pin");
+ PrintRegsList(NoTargets, "No target");
+ };
+
+ // Group candidates per MBB for displaying.
+ DenseMap<const MachineBasicBlock *, SmallVector<const OptRegCandidate *>>
+ CandsPerMBB;
+ for (const Slot &S : Slots) {
+ const OptRegCandidate &Cand = S.Cand;
+ const MachineBasicBlock *MBB =
+ LIS.intervalIsInOneMBB(LIS.getInterval(Cand.Reg.getVirt()));
+ CandsPerMBB[MBB].push_back(&Cand);
+ }
+
+ OS << "* Starting re-assignment phase with registers:\n| Candidates are:\n";
+ OS << " Global (" << CandsPerMBB[nullptr].size() << " registers):\n";
+ PrintRegs(CandsPerMBB[nullptr]);
+ for (const auto &[MBB, Cands] : CandsPerMBB) {
+ if (!MBB)
+ continue;
+ OS << " MBB #" << MBB->getNumber() << " (" << Cands.size()
+ << " registers)\n";
+ PrintRegs(Cands);
+ }
+ });
+}
+
+#endif
+
+namespace {
+class AMDGPUOptimizeVGPREncodingLegacy : public MachineFunctionPass {
+public:
+ static char ID;
+
+ AMDGPUOptimizeVGPREncodingLegacy() : MachineFunctionPass(ID) {}
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ StringRef getPassName() const override {
+ return "AMDGPU Optimize VGPR Encoding";
+ }
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<LiveIntervalsWrapperPass>();
+ AU.addRequired<VirtRegMapWrapperLegacy>();
+ AU.addRequired<LiveRegMatrixWrapperLegacy>();
+ AU.addRequired<SlotIndexesWrapperPass>();
+ AU.addPreserved<LiveIntervalsWrapperPass>();
+ AU.addPreserved<VirtRegMapWrapperLegacy>();
+ AU.addPreserved<LiveRegMatrixWrapperLegacy>();
+ AU.addPreserved<SlotIndexesWrapperPass>();
+ AU.setPreservesCFG();
+ MachineFunctionPass::getAnalysisUsage(AU);
+ }
+};
+} // namespace
+
+INITIALIZE_PASS_BEGIN(AMDGPUOptimizeVGPREncodingLegacy, DEBUG_TYPE,
+ "AMDGPU Optimize VGPR Encoding", false, false)
+INITIALIZE_PASS_DEPENDENCY(LiveIntervalsWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(VirtRegMapWrapperLegacy)
+INITIALIZE_PASS_DEPENDENCY(LiveRegMatrixWrapperLegacy)
+INITIALIZE_PASS_DEPENDENCY(SlotIndexesWrapperPass)
+INITIALIZE_PASS_END(AMDGPUOptimizeVGPREncodingLegacy, DEBUG_TYPE,
+ "AMDGPU Optimize VGPR Encoding", false, false)
+
+char AMDGPUOptimizeVGPREncodingLegacy::ID = 0;
+
+char &llvm::SIAMDGPUOptimizeVGPREncodingLegacyID =
+ AMDGPUOptimizeVGPREncodingLegacy::ID;
+
+bool AMDGPUOptimizeVGPREncodingLegacy::runOnMachineFunction(
+ MachineFunction &MF) {
+ if (skipFunction(MF.getFunction()))
+ return false;
+
+ VirtRegMap &VRM = getAnalysis<VirtRegMapWrapperLegacy>().getVRM();
+ LiveIntervals &LIS = getAnalysis<LiveIntervalsWrapperPass>().getLIS();
+ LiveRegMatrix &LRM = getAnalysis<LiveRegMatrixWrapperLegacy>().getLRM();
+ return AMDGPUOptimizeVGPREncoding(VRM, LIS, LRM).run(MF);
+}
+
+PreservedAnalyses
+AMDGPUOptimizeVGPREncodingPass::run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM) {
+
+ VirtRegMap &VRM = MFAM.getResult<VirtRegMapAnalysis>(MF);
+ LiveIntervals &LIS = MFAM.getResult<LiveIntervalsAnalysis>(MF);
+ LiveRegMatrix &LRM = MFAM.getResult<LiveRegMatrixAnalysis>(MF);
+ if (!AMDGPUOptimizeVGPREncoding(VRM, LIS, LRM).run(MF))
+ return PreservedAnalyses::all();
+
+ return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.h b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.h
new file mode 100644
index 00000000000000..c1e59438929fdf
--- /dev/null
+++ b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.h
@@ -0,0 +1,23 @@
+//===- AMDGPUOptimizeVGPREncoding.h -----------------------------*- C++- *-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUOPTIMIZEVGPRENCODING_H
+#define LLVM_LIB_TARGET_AMDGPU_AMDGPUOPTIMIZEVGPRENCODING_H
+
+#include "llvm/CodeGen/MachinePassManager.h"
+
+namespace llvm {
+class AMDGPUOptimizeVGPREncodingPass
+ : public detail::PassInfoMixin<AMDGPUOptimizeVGPREncodingPass> {
+public:
+ PreservedAnalyses run(MachineFunction &MF,
+ MachineFunctionAnalysisManager &MFAM);
+};
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_AMDGPU_AMDGPUOPTIMIZEVGPRENCODING_H
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
index 372d5f5acab21a..e8e1c61a5cd367 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
+++ b/llvm/lib/Target/AMDGPU/AMDGPUPassRegistry.def
@@ -121,6 +121,7 @@ MACHINE_FUNCTION_PASS("amdgpu-global-isel-divergence-lowering",
MACHINE_FUNCTION_PASS("amdgpu-insert-delay-alu", AMDGPUInsertDelayAluPass())
MACHINE_FUNCTION_PASS("amdgpu-isel", AMDGPUISelDAGToDAGPass(*this))
MACHINE_FUNCTION_PASS("amdgpu-lower-vgpr-encoding", AMDGPULowerVGPREncodingPass())
+MACHINE_FUNCTION_PASS("amdgpu-optimize-vgpr-encoding", AMDGPUOptimizeVGPREncodingPass())
MACHINE_FUNCTION_PASS("amdgpu-mark-last-scratch-load", AMDGPUMarkLastScratchLoadPass())
MACHINE_FUNCTION_PASS("amdgpu-postlegalizer-combiner",
AMDGPUPostLegalizerCombinerPass())
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 4ff5d2295fc6ad..1a88432fa7a5e7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -29,6 +29,7 @@
#include "AMDGPULowerVGPREncoding.h"
#include "AMDGPUMacroFusion.h"
#include "AMDGPUNextUseAnalysis.h"
+#include "AMDGPUOptimizeVGPREncoding.h"
#include "AMDGPUPerfHintAnalysis.h"
#include "AMDGPUPreloadKernArgProlog.h"
#include "AMDGPUPrepareAGPRAlloc.h"
@@ -609,6 +610,11 @@ static cl::opt<bool>
cl::desc("Enable loop data prefetch on AMDGPU"),
cl::Hidden, cl::init(false));
+static cl::opt<bool>
+ EnableVGPREncodingOpt("amdgpu-enable-vgpr-encoding-optimization", cl::Hidden,
+ cl::init(false),
+ cl::desc("Enable VGPR encoding optimization pass"));
+
static cl::opt<std::string>
AMDGPUSchedStrategy("amdgpu-sched-strategy",
cl::desc("Select custom AMDGPU scheduling strategy."),
@@ -709,6 +715,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeAMDGPUTarget() {
initializeSIPeepholeSDWALegacyPass(*PR);
initializeSIShrinkInstructionsLegacyPass(*PR);
initializeSIOptimizeExecMaskingPreRALegacyPass(*PR);
+ initializeAMDGPUOptimizeVGPREncodingLegacyPass(*PR);
initializeSIOptimizeVGPRLiveRangeLegacyPass(*PR);
initializeAMDGPUNextUseAnalysisLegacyPassPass(*PR);
initializeAMDGPUNextUseAnalysisPrinterLegacyPassPass(*PR);
@@ -2006,6 +2013,9 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
// For allocating per-thread VGPRs.
addPass(createVGPRAllocPass(true));
+ if (EnableVGPREncodingOpt)
+ addPass(&SIAMDGPUOptimizeVGPREncodingLegacyID);
+
addPreRewrite();
addPass(&VirtRegRewriterID);
@@ -2742,6 +2752,9 @@ Expected<bool> AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
else
addMachineFunctionPass(RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
+ if (EnableVGPREncodingOpt)
+ addMachineFunctionPass(AMDGPUOptimizeVGPREncodingPass(), PMW);
+
addPreRewrite(PMW);
addMachineFunctionPass(VirtRegRewriterPass(true), PMW);
diff --git a/llvm/lib/Target/AMDGPU/CMakeLists.txt b/llvm/lib/Target/AMDGPU/CMakeLists.txt
index 4a5f77d55afa5f..eb37ea997968b9 100644
--- a/llvm/lib/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/lib/Target/AMDGPU/CMakeLists.txt
@@ -90,6 +90,7 @@ add_llvm_target(AMDGPUCodeGen
AMDGPUCoExecSchedStrategy.cpp
AMDGPUIGroupLP.cpp
AMDGPULowerVGPREncoding.cpp
+ AMDGPUOptimizeVGPREncoding.cpp
AMDGPUMCResourceInfo.cpp
AMDGPUMarkLastScratchLoad.cpp
AMDGPUMIRFormatter.cpp
>From 060eaca22591b7b563c82c3cd67f3269eaab0844 Mon Sep 17 00:00:00 2001
From: Lucas Ramirez <lucas.rami at proton.me>
Date: Thu, 10 Sep 2026 17:08:42 +0000
Subject: [PATCH 2/5] Fix typos, add missing analyses, enable pass by default
on -O3
---
.../AMDGPU/AMDGPUOptimizeVGPREncoding.cpp | 194 ++++++++++--------
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 9 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 1 +
3 files changed, 106 insertions(+), 98 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
index f19911ce6b797a..16733e92a825a4 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
@@ -1,4 +1,4 @@
-//===-- AMDGPUOptimizeVGPREncoding.cpp --------------------------*- C++- *-===//
+//===-- AMDGPUOptimizeVGPREncoding.cpp --------------------------*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
@@ -16,7 +16,7 @@
/// occur.
///
/// In the future, the intent is for this pass to also try to minimize VGPR bank
-/// conflicts on subtarget where it is relevant.
+/// conflicts on subtargets where it is relevant.
//
//===----------------------------------------------------------------------===//
@@ -30,8 +30,10 @@
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallBitVector.h"
+#include "llvm/CodeGen/LiveDebugVariables.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/LiveRegMatrix.h"
+#include "llvm/CodeGen/LiveStacks.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -75,10 +77,10 @@ struct ModeInstr {
/// of a previous/next instruction.
static constexpr unsigned NoIdx = ~0U;
- /// For each operand type, virtual of physical VGPR operand used by the
+ /// For each operand type, virtual or physical VGPR operand used by the
/// instruction. A null register indicates the instruction has no VGPR operand
/// of that type. For VOPD instructions, this holds VGPRs for the first of the
- /// two instruction which define a VGPR of each operand type.
+ /// two instructions which define a VGPR of each operand type.
std::array<Register, NumOprdTypes> Oprds;
/// For each operand type, indices of previous/next neighbor instructions with
/// defined operands in the instruction list this instruction is a part of.
@@ -178,8 +180,8 @@ class MBBModeUsage {
/// currently in the corresponding MSB group. Two registers (virtual or
/// physical) are initially considered neighbors when they are used as operands
/// of the same type in two neighboring instructions (c.f. \ref ModeInstr). A
-/// register's neighborhood---and thefore its score---changes throughout the
-/// pass's lifetime to reflect the simulataed placement of MODE-setting
+/// register's neighborhood---and therefore its score---changes throughout the
+/// pass's lifetime to reflect the simulated placement of MODE-setting
/// instructions.
///
/// An optimizable register is said to be "pinned" when the MSB group of its
@@ -190,9 +192,9 @@ class OptReg {
public:
using WeightedNeighbors = SmallDenseMap<OptReg *, unsigned, 4>;
- /// Abstract coordinates for an occurence of this register.
+ /// Abstract coordinates for an occurrence of this register.
struct Coordinates {
- /// This index of the MBB.
+ /// The index of the MBB.
unsigned MBBIndex;
/// The index of the MODE-using instruction.
unsigned InstrIdx;
@@ -205,18 +207,18 @@ class OptReg {
/// created/destroyed through class methods.
OptReg(Register VirtReg, const VirtRegMap &VRM);
- /// Returns the total number of occurences of pinned neighbor registers in \p
+ /// Returns the total number of occurrences of pinned neighbor registers in \p
/// Group.
unsigned getGroupPinnedScore(MSBGroup Group) const {
return PinnedScore[Group];
}
- /// Returns the total number of occurences of neighbor registers in \p Group.
+ /// Returns the total number of occurrences of neighbor registers in \p Group.
unsigned getGroupScore(MSBGroup Group) const {
return PinnedScore[Group] + Score[Group];
}
- /// Returns the total number of occurences of neighbor registers in this
+ /// Returns the total number of occurrences of neighbor registers in this
/// register's current MSB group.
unsigned getCurrentGroupScore() const { return getGroupScore(MSB); }
@@ -229,8 +231,8 @@ class OptReg {
const WeightedNeighbors &getNeighbors() const { return Neighbors; }
/// Returns the list of coordinates corresponding to this register's
- /// occurences.
- ArrayRef<Coordinates> getOccurences() const { return Occurences; }
+ /// occurrences.
+ ArrayRef<Coordinates> getOccurrences() const { return Occurrences; }
/// Returns the underlying virtual register.
Register getVirt() const { return VirtReg; }
@@ -245,23 +247,23 @@ class OptReg {
/// Returns whether the register is pinned.
bool isPinned() const { return IsPinned; }
- // addOccurence and record* methods used by OptimizableRegs to initialize the
- // occurences and neighborhood of all optimizable registers at the beginning.
+ // addOccurrence and record* methods used by OptimizableRegs to initialize the
+ // occurrences and neighborhood of all optimizable registers at the beginning.
- /// Adds an occurence of this register in operand type \p Oprd of instruction
+ /// Adds an occurrence of this register in operand type \p Oprd of instruction
/// \p InstrIdx of MBB \p MBBIdx.
- void addOccurence(unsigned MBBIndex, unsigned InstrIdx, OprdType Oprd) {
- Occurences.push_back({MBBIndex, InstrIdx, Oprd});
+ void addOccurrence(unsigned MBBIndex, unsigned InstrIdx, OprdType Oprd) {
+ Occurrences.push_back({MBBIndex, InstrIdx, Oprd});
}
- /// Records an occurence of \p NeighborReg as a neighbor.
- void recordNeighborOccurence(OptReg &NeighborReg);
+ /// Records an occurrence of \p NeighborReg as a neighbor.
+ void recordNeighborOccurrence(OptReg &NeighborReg);
- /// Records an occurence of physical register \p PhysReg as a neighbor.
- void recordPhysNeighborOccurence(Register PhysReg, const VirtRegMap &VRM);
+ /// Records an occurrence of physical register \p PhysReg as a neighbor.
+ void recordPhysNeighborOccurrence(Register PhysReg, const VirtRegMap &VRM);
- /// Records an occurence of this register at a block boundary. This adds a
- /// "pinned occurence" of the default MSB group in which all MBBs start and
+ /// Records an occurrence of this register at a block boundary. This adds a
+ /// "pinned occurrence" of the default MSB group in which all MBBs start and
/// end.
void recordBlockBoundaryPin() { ++PinnedScore[DefaultGroup]; }
@@ -270,25 +272,25 @@ class OptReg {
// simulates placement of MODE-setting instructions. remove* methods mirror
// record* methods 1-to-1.
- /// Pins this register's to the MSB group of its currently assigned physical
+ /// Pins this register to the MSB group of its currently assigned physical
/// register.
void pinMSBGroup();
- /// Removes an occurence of \p NeighborReg as a neighbor.
- void removeNeighborOccurence(OptReg &NeighborReg);
+ /// Removes an occurrence of \p NeighborReg as a neighbor.
+ void removeNeighborOccurrence(OptReg &NeighborReg);
- /// Removes an occurence of physical register \p PhysReg as a neighbor.
- void removePhysNeighborOccurence(Register PhysReg, const VirtRegMap &VRM);
+ /// Removes an occurrence of physical register \p PhysReg as a neighbor.
+ void removePhysNeighborOccurrence(Register PhysReg, const VirtRegMap &VRM);
- /// Removes an occurence of this register at a block boundary. This removes a
- /// "pinned occurence" of the default MSB group in which all MBBs start and
+ /// Removes an occurrence of this register at a block boundary. This removes a
+ /// "pinned occurrence" of the default MSB group in which all MBBs start and
/// end.
void removeBlockBoundaryPin() {
assert(PinnedScore[DefaultGroup] > 0 && "underflow");
--PinnedScore[DefaultGroup];
}
- /// Notifies the optimizable register that its assigned physcial register has
+ /// Notifies the optimizable register that its assigned physical register has
/// changed and that its new assignment belongs to MSB group \p NewGroup. It
/// is illegal to change the MSB group of a pinned register.
void notifyPhysAssignmentChanged(MSBGroup NewGroup);
@@ -300,21 +302,21 @@ class OptReg {
private:
/// The virtual register.
Register VirtReg;
- /// MSB group of the virtual register's current physcial register assignment.
+ /// MSB group of the virtual register's current physical register assignment.
MSBGroup MSB;
- /// Per-MSB group score, counting the number of occurences of neighbor
- /// registers in each group, separated between occurences of unpinned
+ /// Per-MSB group score, counting the number of occurrences of neighbor
+ /// registers in each group, separated between occurrences of unpinned
/// optimizable registers from the others (physical registers, pinned
/// optimizable registers, and block boundaries). Reflects the register's
/// current neighborhood.
std::array<unsigned, NumMSBGroups> Score, PinnedScore;
/// Maps neighboring optimizable registers to the number of times they occur
- /// in the neighborhood of one of this register's occurences. Neighbors can be
- /// added or removed at will after construction, impacting the score.
+ /// in the neighborhood of one of this register's occurrences. Neighbors can
+ /// be added or removed at will after construction, impacting the score.
WeightedNeighbors Neighbors;
- /// Occurences of this register in the function under consideration.
- /// Occurences can be added after construction but cannot be removed.
- SmallVector<Coordinates> Occurences;
+ /// Occurrences of this register in the function under consideration.
+ /// Occurrences can be added after construction but cannot be removed.
+ SmallVector<Coordinates> Occurrences;
/// Whether the register is pinned to \ref MSB.
bool IsPinned = false;
};
@@ -370,22 +372,22 @@ class OptimizableRegs {
SmallVector<unsigned, 0> VirtRegToStorageIdx;
};
-/// Simulates placement of MODE-setting instructions as unoptimizable MSB groups
+/// Simulates placement of MODE-setting instructions as unoptimizable MSB group
/// conflicts are detected, driving optimization forward by progressively
/// pruning register neighborhoods and pinning optimizable registers once they
/// reach an "ideal" MSB group.
///
/// The detection and resolution of unoptimizable MSB group conflicts is this
/// class's main purpose. An optimizable register with non-null score
-/// contributions from pinned neighbors in more that one MSB group will
-/// necessarily require MODE-setting instructions around its occurences that
+/// contributions from pinned neighbors in more than one MSB group will
+/// necessarily require MODE-setting instructions around its occurrences that
/// neighbor pinned registers in all but one of those MSB groups. This is a
/// conflict in the sense that we would need the register to be in multiple MSB
/// groups at the same time to not need MODE-setting instructions. It is
/// unoptimizable by the pass because pinned neighbors are not allowed to change
/// MSB group, so no amount of register re-assignment can resolve it. The
/// objective is to detect those situations early so that no effort is made
-/// attempting to optimize MSB conflits at code locations where we are
+/// attempting to optimize MSB conflicts at code locations where we are
/// guaranteed to be unable to solve them. The class resolves such conflicts by
/// simulating the placement of MODE-setting instructions around problematic
/// registers, effectively "breaking" their relationships with some pinned
@@ -399,9 +401,9 @@ class OptimizableRegs {
/// registers reach an "ideal" MSB group that they can be pinned to.
///
/// FIXME: The current approach to determine where we place MODE-setting
-/// instructions to resolve conflicts is correct however when there are multiple
-/// possible locations to choose from it does not attempt to analyze the
-/// expected benefit of each. Picking the best location in such cases should
+/// instructions to resolve conflicts is correct, however when there are
+/// multiple possible locations to choose from it does not attempt to analyze
+/// the expected benefit of each. Picking the best location in such cases should
/// improve overall pass performance.
class ModeSetOptimizer {
public:
@@ -446,8 +448,8 @@ class ModeSetOptimizer {
ArrayRef<MBBModeUsage> ModeUsage;
const VirtRegMap &VRM;
- /// Resolve conflicts for \p Reg, if any, and returns whether the register had
- /// conflicts.
+ /// Resolves conflicts for \p Reg, if any, and returns whether the register
+ /// had conflicts.
bool resolveConflictingPins(OptReg &Reg);
/// Returns whether we consider that \p Reg should be pinned. Registers whose
@@ -455,7 +457,7 @@ class ModeSetOptimizer {
/// group are in the perfect MSB group and should never change group again.
bool shouldBePinned(const OptReg &Reg) const;
- /// Pins \p Reg if it is eligible according to \ref shouldbePinned. Returns
+ /// Pins \p Reg if it is eligible according to \ref shouldBePinned. Returns
/// whether the register was newly pinned.
bool pinIfEligible(OptReg &Reg);
@@ -633,7 +635,7 @@ class MaxHeap {
/// parent. Returns true if it moved at all.
bool siftUp(unsigned HeapIdx);
- /// Sifts the element at \p HeapIdx towards the leaves while it is outranged
+ /// Sifts the element at \p HeapIdx towards the leaves while it is outranked
/// by its children.
bool siftDown(unsigned HeapIdx);
};
@@ -667,7 +669,7 @@ class AMDGPUOptimizeVGPREncoding {
public:
AMDGPUOptimizeVGPREncoding(VirtRegMap &VRM, LiveIntervals &LIS,
LiveRegMatrix &LRM)
- : VRM(VRM), LIS(LIS), LRM(LRM){};
+ : VRM(VRM), LIS(LIS), LRM(LRM) {}
bool run(MachineFunction &MF);
@@ -875,7 +877,7 @@ void OptReg::pinMSBGroup() {
}
}
-void OptReg::recordNeighborOccurence(OptReg &NeighborReg) {
+void OptReg::recordNeighborOccurrence(OptReg &NeighborReg) {
assert(&NeighborReg != this && "cannot be neighbor with itself");
++Neighbors.insert({&NeighborReg, 0}).first->getSecond();
if (NeighborReg.isPinned())
@@ -884,7 +886,7 @@ void OptReg::recordNeighborOccurence(OptReg &NeighborReg) {
++Score[NeighborReg.MSB];
}
-void OptReg::removeNeighborOccurence(OptReg &NeighborReg) {
+void OptReg::removeNeighborOccurrence(OptReg &NeighborReg) {
// Update neighbors.
auto Neighbor = Neighbors.find(&NeighborReg);
assert(Neighbor != Neighbors.end() && "neighbor must exist");
@@ -901,14 +903,14 @@ void OptReg::removeNeighborOccurence(OptReg &NeighborReg) {
}
}
-void OptReg::recordPhysNeighborOccurence(Register PhysReg,
- const VirtRegMap &VRM) {
+void OptReg::recordPhysNeighborOccurrence(Register PhysReg,
+ const VirtRegMap &VRM) {
assert(PhysReg.isPhysical() && "must be physical register");
++PinnedScore[getVGPRGroup(PhysReg, VRM)];
}
-void OptReg::removePhysNeighborOccurence(Register PhysReg,
- const VirtRegMap &VRM) {
+void OptReg::removePhysNeighborOccurrence(Register PhysReg,
+ const VirtRegMap &VRM) {
assert(PhysReg.isPhysical() && "must be physical register");
MSBGroup PhysGroup = getVGPRGroup(PhysReg, VRM);
assert(PinnedScore[PhysGroup] > 0 && "underflow");
@@ -947,8 +949,8 @@ OptimizableRegs::OptimizableRegs(const BitVector &OptVirtRegs,
Reg.pinMSBGroup();
}
- // Idnetify the neighborhood and occurences of each register. This initializes
- // the score of all optimizable registers.
+ // Identify the neighborhood and occurrences of each register. This
+ // initializes the score of all optimizable registers.
for (const auto &[MBBIdx, BlockUsage] : enumerate(ModeUsage)) {
ArrayRef<ModeInstr> Instructions = BlockUsage.getInstructions();
if (Instructions.empty())
@@ -965,7 +967,7 @@ OptimizableRegs::OptimizableRegs(const BitVector &OptVirtRegs,
OptReg *CurrentOptReg = (*this)[Reg];
if (CurrentOptReg) {
assert(Reg.isVirtual() && "only virtregs are optimizable");
- CurrentOptReg->addOccurence(MBBIdx, InstrIdx, Oprd);
+ CurrentOptReg->addOccurrence(MBBIdx, InstrIdx, Oprd);
if (!PreviousOptReg) {
unsigned PrevIdx = CurrentInstr.Prev[Oprd];
@@ -976,19 +978,19 @@ OptimizableRegs::OptimizableRegs(const BitVector &OptVirtRegs,
// the first register.
CurrentOptReg->recordBlockBoundaryPin();
} else {
- // The register immediately preceeding this was a physical one.
+ // The register immediately preceding this was a physical one.
Register PhysReg = Instructions[PrevIdx].Oprds[Oprd];
- CurrentOptReg->recordPhysNeighborOccurence(PhysReg, VRM);
+ CurrentOptReg->recordPhysNeighborOccurrence(PhysReg, VRM);
}
} else if (CurrentOptReg != PreviousOptReg) {
// The two virtual registers are neighbors.
- CurrentOptReg->recordNeighborOccurence(*PreviousOptReg);
- PreviousOptReg->recordNeighborOccurence(*CurrentOptReg);
+ CurrentOptReg->recordNeighborOccurrence(*PreviousOptReg);
+ PreviousOptReg->recordNeighborOccurrence(*CurrentOptReg);
}
} else if (PreviousOptReg) {
// We have a virtual register followed by a physical one on the same
// operand stream. We just need to update the former's pin score.
- PreviousOptReg->recordPhysNeighborOccurence(Reg, VRM);
+ PreviousOptReg->recordPhysNeighborOccurrence(Reg, VRM);
PreviousOptReg = nullptr;
}
@@ -1043,7 +1045,7 @@ MSBGroup ModeSetOptimizer::selectPreferredMSBGroup(const OptReg &Reg) const {
if (PinnedScore < MaxPinnedScore)
continue;
- // Ammong MSB groups with the same number of pinned neighbors, favor the one
+ // Among MSB groups with the same number of pinned neighbors, favor the one
// with highest number of unpinned neighbors.
unsigned TotalScore = Reg.getGroupScore(Group);
if (PinnedScore == MaxPinnedScore && MaxTotalScore > TotalScore)
@@ -1112,21 +1114,21 @@ void ModeSetOptimizer::breakNeighborRelationship(unsigned MBBIdx,
if (AfterReg == BeforeReg)
return;
- // Notify registers that one occurence of their neighborhood relationship is
+ // Notify registers that one occurrence of their neighborhood relationship is
// broken. Optimizable registers which have their score affected by the
// break may become pinnable.
- OptReg *AferOptReg = OptRegs[AfterReg];
+ OptReg *AfterOptReg = OptRegs[AfterReg];
OptReg *BeforeOptReg = OptRegs[BeforeReg];
- if (AferOptReg && BeforeOptReg) {
- AferOptReg->removeNeighborOccurence(*BeforeOptReg);
- BeforeOptReg->removeNeighborOccurence(*AferOptReg);
+ if (AfterOptReg && BeforeOptReg) {
+ AfterOptReg->removeNeighborOccurrence(*BeforeOptReg);
+ BeforeOptReg->removeNeighborOccurrence(*AfterOptReg);
CheckShouldBePinned.set(AfterReg.virtRegIndex());
CheckShouldBePinned.set(BeforeReg.virtRegIndex());
- } else if (AferOptReg) {
- AferOptReg->removePhysNeighborOccurence(BeforeReg, VRM);
+ } else if (AfterOptReg) {
+ AfterOptReg->removePhysNeighborOccurrence(BeforeReg, VRM);
CheckShouldBePinned.set(AfterReg.virtRegIndex());
} else if (BeforeOptReg) {
- BeforeOptReg->removePhysNeighborOccurence(AfterReg, VRM);
+ BeforeOptReg->removePhysNeighborOccurrence(AfterReg, VRM);
CheckShouldBePinned.set(BeforeReg.virtRegIndex());
}
}
@@ -1148,7 +1150,7 @@ bool ModeSetOptimizer::resolveConflictingPins(OptReg &Reg) {
// group is not the preferred one.
if (PreferredGroup != DefaultGroup &&
Reg.getGroupPinnedScore(DefaultGroup) > 0) {
- for (const auto &[MBBIdx, InstrIdx, Oprd] : Reg.getOccurences()) {
+ for (const auto &[MBBIdx, InstrIdx, Oprd] : Reg.getOccurrences()) {
const MBBModeUsage &MBB = ModeUsage[MBBIdx];
if (InstrIdx == MBB.getFirstOprd(Oprd) &&
!hasModeSetBefore(MBBIdx, InstrIdx)) {
@@ -1167,25 +1169,25 @@ bool ModeSetOptimizer::resolveConflictingPins(OptReg &Reg) {
}
}
- // Conflicts with occurences of pinned neighbors in the non-preferred MSB
+ // Conflicts with occurrences of pinned neighbors in the non-preferred MSB
// group need to be resolved. Iterate over a copy of the list of neighbors
// because they will be modified as we simulate placement of MODE-setting
// instructions.
OptReg::WeightedNeighbors Neighbors(Reg.getNeighbors());
for (const auto &[NeighborReg, _] : Neighbors) {
- // Early exit when we know we are not gonna find conflicting occurences.
+ // Early exit when we know we are not going to find conflicting occurrences.
if (!NeighborReg->isPinned() || NeighborReg->getMSB() == PreferredGroup ||
Reg.getGroupPinnedScore(NeighborReg->getMSB()) == 0)
continue;
- // Look through occurences for conflicts.
- for (const auto &[MBBIdx, InstrIdx, Oprd] : NeighborReg->getOccurences()) {
+ // Look through occurrences for conflicts.
+ for (const auto &[MBBIdx, InstrIdx, Oprd] : NeighborReg->getOccurrences()) {
const MBBModeUsage &MBB = ModeUsage[MBBIdx];
ArrayRef<ModeInstr> Instructions = MBB.getInstructions();
- // Look at the operand immediately before this neighbor occurence. If it
+ // Look at the operand immediately before this neighbor occurrence. If it
// matches the register for which we are currently resolving conflicts,
- // then it is one of the neighborhood relationship to break.
+ // then it is one of the neighborhood relationships to break.
unsigned PrevIdx = MBB.getLastInstrBefore(InstrIdx, Oprd);
if (PrevIdx != ModeInstr::NoIdx &&
Instructions[PrevIdx].Oprds[Oprd] == Reg.getVirt() &&
@@ -1198,9 +1200,9 @@ bool ModeSetOptimizer::resolveConflictingPins(OptReg &Reg) {
placeJustBefore(MBBIdx, InstrIdx);
}
- // Look at the operand immediately after this neighbor occurence. If it
+ // Look at the operand immediately after this neighbor occurrence. If it
// matches the register for which we are currently resolving conflicts,
- // then it is one of the neighborhood relationship to break.
+ // then it is one of the neighborhood relationships to break.
unsigned NextIdx = MBB.getFirstInstrAfter(InstrIdx, Oprd);
if (NextIdx != ModeInstr::NoIdx &&
Instructions[NextIdx].Oprds[Oprd] == Reg.getVirt() &&
@@ -1332,7 +1334,7 @@ void OptRegCandidate::update() {
// per-group score, at the extreme registers whose neighborhood is entirely
// assigned to a single MSB group. On the contrary, registers whose
// neighborhood is roughly evenly split between all MSB groups are not that
- // profitable to re-assign; even if a MSB group with higher score than the
+ // profitable to re-assign, even if an MSB group with a higher score than the
// current one the register is in exists.
Benefit = CurrentScore;
unsigned MaxGroupScore = CurrentScore;
@@ -1365,7 +1367,7 @@ void OptRegCandidate::update() {
return;
// We want to favor registers which would benefit from being re-assigned to
- // the single MSB group in which they haved pinned neighbors (if there is
+ // the single MSB group in which they have pinned neighbors (if there is
// one), as this is likely to resolve conflicts in the future without having
// to place MODE-setting instructions.
if (PinGroups.none()) {
@@ -1539,9 +1541,9 @@ bool AMDGPUOptimizeVGPREncoding::run(MachineFunction &MF) {
// Constructs a max-heap with all remaining unpinned registers that are
// candidates for re-assignment.
//
- /// FIXME: There is no need to re-construct the heap every time, we can just
- /// let pin registers fall to the bottom of it since they have no target MSB
- /// group by construction.
+ // FIXME: There is no need to re-construct the heap every time, we can just
+ // let pinned registers fall to the bottom of it since they have no target
+ // MSB group by construction.
MaxHeap AllCandidates(OptRegs);
LLVM_DEBUG(dbgs() << AllCandidates.print(VRM, LIS));
@@ -1554,7 +1556,7 @@ bool AMDGPUOptimizeVGPREncoding::run(MachineFunction &MF) {
BitVector ScoreChanged(OptRegs.getNumVirtRegs());
// Only accepting profitable candidates guarantees forward progress
- // because re-assigning to target groups increase the combined score, which
+ // because re-assigning to target groups increases the combined score, which
// is upper-bounded by the number of neighboring relationships between all
// registers (itself only decreasing after initialization).
while (OptRegCandidate *Candidate = AllCandidates.getMostProfitable()) {
@@ -1790,6 +1792,8 @@ class AMDGPUOptimizeVGPREncodingLegacy : public MachineFunctionPass {
AU.addPreserved<VirtRegMapWrapperLegacy>();
AU.addPreserved<LiveRegMatrixWrapperLegacy>();
AU.addPreserved<SlotIndexesWrapperPass>();
+ AU.addPreserved<LiveDebugVariablesWrapperLegacy>();
+ AU.addPreserved<LiveStacksWrapperLegacy>();
AU.setPreservesCFG();
MachineFunctionPass::getAnalysisUsage(AU);
}
@@ -1831,5 +1835,13 @@ AMDGPUOptimizeVGPREncodingPass::run(MachineFunction &MF,
if (!AMDGPUOptimizeVGPREncoding(VRM, LIS, LRM).run(MF))
return PreservedAnalyses::all();
- return getMachineFunctionPassPreservedAnalyses().preserveSet<CFGAnalyses>();
+ auto PA = getMachineFunctionPassPreservedAnalyses();
+ PA.preserveSet<CFGAnalyses>();
+ PA.preserve<LiveIntervalsAnalysis>();
+ PA.preserve<SlotIndexesAnalysis>();
+ PA.preserve<LiveDebugVariablesAnalysis>();
+ PA.preserve<LiveStacksAnalysis>();
+ PA.preserve<VirtRegMapAnalysis>();
+ PA.preserve<LiveRegMatrixAnalysis>();
+ return PA;
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 1a88432fa7a5e7..24c291daa6e4ce 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -610,11 +610,6 @@ static cl::opt<bool>
cl::desc("Enable loop data prefetch on AMDGPU"),
cl::Hidden, cl::init(false));
-static cl::opt<bool>
- EnableVGPREncodingOpt("amdgpu-enable-vgpr-encoding-optimization", cl::Hidden,
- cl::init(false),
- cl::desc("Enable VGPR encoding optimization pass"));
-
static cl::opt<std::string>
AMDGPUSchedStrategy("amdgpu-sched-strategy",
cl::desc("Select custom AMDGPU scheduling strategy."),
@@ -2013,7 +2008,7 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
// For allocating per-thread VGPRs.
addPass(createVGPRAllocPass(true));
- if (EnableVGPREncodingOpt)
+ if (getOptLevel() >= CodeGenOptLevel::Aggressive)
addPass(&SIAMDGPUOptimizeVGPREncodingLegacyID);
addPreRewrite();
@@ -2752,7 +2747,7 @@ Expected<bool> AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
else
addMachineFunctionPass(RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
- if (EnableVGPREncodingOpt)
+ if (getOptLevel() >= CodeGenOptLevel::Aggressive)
addMachineFunctionPass(AMDGPUOptimizeVGPREncodingPass(), PMW);
addPreRewrite(PMW);
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index df1d419bea1d73..1a88b30f9b4120 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -1428,6 +1428,7 @@
; GCN-O3-NEXT: Virtual Register Map
; GCN-O3-NEXT: Live Register Matrix
; GCN-O3-NEXT: Greedy Register Allocator
+; GCN-O3-NEXT: AMDGPU Optimize VGPR Encoding
; GCN-O3-NEXT: GCN NSA Reassign
; GCN-O3-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
; GCN-O3-NEXT: Virtual Register Rewriter
>From 2ed96d63aed6e0b38d6ef46305fa5ed9c6cd75f1 Mon Sep 17 00:00:00 2001
From: Lucas Ramirez <lucas.rami at proton.me>
Date: Tue, 15 Sep 2026 16:48:25 +0000
Subject: [PATCH 3/5] Enable on all opt-levels / unit-tests
---
.../AMDGPU/AMDGPUOptimizeVGPREncoding.cpp | 34 +-
.../lib/Target/AMDGPU/AMDGPUTargetMachine.cpp | 8 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline.ll | 5 +-
.../CodeGen/AMDGPU/sgpr-regalloc-flags.ll | 4 +
.../AMDGPU/AMDGPUOptimizeVGPREncodingTest.cpp | 407 ++++++++++++++++++
llvm/unittests/Target/AMDGPU/CMakeLists.txt | 1 +
6 files changed, 429 insertions(+), 30 deletions(-)
create mode 100644 llvm/unittests/Target/AMDGPU/AMDGPUOptimizeVGPREncodingTest.cpp
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
index 16733e92a825a4..94929dba566c41 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
@@ -59,6 +59,9 @@ using MSBGroup = unsigned;
static constexpr unsigned NumMSBGroups = 4;
static constexpr unsigned DefaultGroup = 0;
+/// Groups are 256 registers each.
+static constexpr unsigned GroupSizeLog2 = 8;
+
/// Operand type where the MSB group is relevant, identified by an unsigned ID
/// in [0, NumOprdTypes).
using OprdType = unsigned;
@@ -662,7 +665,7 @@ class VirtRegReMap {
RegisterClassInfo RCI;
const VirtRegMap &VRM;
const SIRegisterInfo &TRI;
- const BitVector ReservedRegs;
+ const BitVector &ReservedRegs;
};
class AMDGPUOptimizeVGPREncoding {
@@ -697,7 +700,7 @@ static MSBGroup getVGPRGroup(Register Reg, const VirtRegMap &VRM) {
const auto &TRI =
*static_cast<const SIRegisterInfo *>(&VRM.getTargetRegInfo());
MCRegister PhysReg = Reg.isVirtual() ? VRM.getPhys(Reg) : Reg.asMCReg();
- return TRI.getHWRegIndex(PhysReg) >> 8;
+ return TRI.getHWRegIndex(PhysReg) >> GroupSizeLog2;
}
/// Returns true if \p RC is confined to the first 256 VGPRs i.e., every
@@ -806,7 +809,7 @@ MBBModeUsage::MBBModeUsage(const MachineBasicBlock &MBB, BitVector &OptVirtRegs,
if (!VOPDTable)
continue;
for (OprdType Oprd : seq(NumOprdTypes)) {
- const MachineOperand *MO = GetRelevantMO(Table[Oprd]);
+ const MachineOperand *MO = GetRelevantMO(VOPDTable[Oprd]);
if (!MO)
continue;
Register Reg = MO->getReg();
@@ -1450,7 +1453,7 @@ VirtRegReMap::VirtRegReMap(LiveRegMatrix &LRM, LiveIntervals &LIS,
const MachineFunction &MF, const VirtRegMap &VRM)
: LRM(LRM), LIS(LIS), VRM(VRM),
TRI(*static_cast<const SIRegisterInfo *>(&VRM.getTargetRegInfo())),
- ReservedRegs(TRI.getReservedRegs(MF)) {
+ ReservedRegs(MF.getRegInfo().getReservedRegs()) {
RCI.runOnMachineFunction(MF);
}
@@ -1458,7 +1461,7 @@ static bool overflowsLastGroup(unsigned HWRegIdx, unsigned NumLanes,
MSBGroup Group) {
if (Group != NumMSBGroups - 1)
return false;
- return ((HWRegIdx + NumLanes - 1) >> 8) != NumMSBGroups - 1;
+ return ((HWRegIdx + NumLanes - 1) >> GroupSizeLog2) != NumMSBGroups - 1;
}
MCRegister VirtRegReMap::tryAssignInGroup(Register VirtReg, MSBGroup Group) {
@@ -1582,7 +1585,9 @@ bool AMDGPUOptimizeVGPREncoding::run(MachineFunction &MF) {
// Attempts re-assignment to a better MSB group.
if (VRRM.tryAssignToTargetGroups(Candidate->Reg, Candidate->Targets)) {
LLVM_DEBUG(dbgs() << " | SUCCESS: Assigned to free physical register "
- << VRM.getPhys(Candidate->Reg.getVirt()) << '\n');
+ << printReg(VRM.getPhys(Candidate->Reg.getVirt()),
+ ST.getRegisterInfo(), 0, &MF.getRegInfo())
+ << '\n');
++Epoch;
Optimizer.regChangedGroup(Candidate->Reg, ScoreChanged);
}
@@ -1608,9 +1613,6 @@ Printable MBBModeUsage::print(const VirtRegMap &VRM) const {
return Printable([&](raw_ostream &OS) {
const MachineRegisterInfo &MRI = VRM.getRegInfo();
- std::array<std::optional<MSBGroup>, NumOprdTypes> GroupPerOprd;
- GroupPerOprd.fill(0);
-
// Display operand type names.
OS << " [ ";
for (OprdType Oprd : seq(NumOprdTypes)) {
@@ -1639,20 +1641,6 @@ Printable MBBModeUsage::print(const VirtRegMap &VRM) const {
StringRef RegStr = RegStream.str();
RegStr = RegStr.take_front(ColumnWidth);
OS << RegStr << indent(ColumnWidth - RegStr.size());
-
- MSBGroup OprdGroup = getVGPRGroup(Reg, VRM);
- std::optional<MSBGroup> &CurrentOprdGroup = GroupPerOprd[Oprd];
- if (CurrentOprdGroup.has_value()) {
- if (*CurrentOprdGroup != OprdGroup) {
- // Different group as previously used for that operand.
- CurrentOprdGroup = OprdGroup;
- }
- } else {
- // First operand in the sequence. This defines the initial desired
- // group for that operand in the instruction sequence, so we do not
- // account for the set MSB here.
- CurrentOprdGroup = OprdGroup;
- }
}
OS << " ]\n";
}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
index 24c291daa6e4ce..4caf4b90cd2bc9 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetMachine.cpp
@@ -1892,6 +1892,7 @@ bool GCNPassConfig::addPreRewrite() {
addPass(&GCNNSAReassignID);
addPass(&AMDGPURewriteAGPRCopyMFMALegacyID);
+ addPass(&SIAMDGPUOptimizeVGPREncodingLegacyID);
return true;
}
@@ -2008,9 +2009,6 @@ bool GCNPassConfig::addRegAssignAndRewriteOptimized() {
// For allocating per-thread VGPRs.
addPass(createVGPRAllocPass(true));
- if (getOptLevel() >= CodeGenOptLevel::Aggressive)
- addPass(&SIAMDGPUOptimizeVGPREncodingLegacyID);
-
addPreRewrite();
addPass(&VirtRegRewriterID);
@@ -2591,6 +2589,7 @@ void AMDGPUCodeGenPassBuilder::addPreRewrite(PassManagerWrapper &PMW) {
}
addMachineFunctionPass(AMDGPURewriteAGPRCopyMFMAPass(), PMW);
+ addMachineFunctionPass(AMDGPUOptimizeVGPREncodingPass(), PMW);
}
void AMDGPUCodeGenPassBuilder::addMachineSSAOptimization(
@@ -2747,9 +2746,6 @@ Expected<bool> AMDGPUCodeGenPassBuilder::addRegAssignAndRewriteOptimized(
else
addMachineFunctionPass(RAGreedyPass({onlyAllocateVGPRs, "vgpr"}), PMW);
- if (getOptLevel() >= CodeGenOptLevel::Aggressive)
- addMachineFunctionPass(AMDGPUOptimizeVGPREncodingPass(), PMW);
-
addPreRewrite(PMW);
addMachineFunctionPass(VirtRegRewriterPass(true), PMW);
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
index 1a88b30f9b4120..4236f2c8d84893 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline.ll
@@ -409,6 +409,7 @@
; GCN-O1-NEXT: Greedy Register Allocator
; GCN-O1-NEXT: GCN NSA Reassign
; GCN-O1-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
+; GCN-O1-NEXT: AMDGPU Optimize VGPR Encoding
; GCN-O1-NEXT: Virtual Register Rewriter
; GCN-O1-NEXT: AMDGPU Mark Last Scratch Load
; GCN-O1-NEXT: Stack Slot Coloring
@@ -741,6 +742,7 @@
; GCN-O1-OPTS-NEXT: Greedy Register Allocator
; GCN-O1-OPTS-NEXT: GCN NSA Reassign
; GCN-O1-OPTS-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
+; GCN-O1-OPTS-NEXT: AMDGPU Optimize VGPR Encoding
; GCN-O1-OPTS-NEXT: Virtual Register Rewriter
; GCN-O1-OPTS-NEXT: AMDGPU Mark Last Scratch Load
; GCN-O1-OPTS-NEXT: Stack Slot Coloring
@@ -1078,6 +1080,7 @@
; GCN-O2-NEXT: Greedy Register Allocator
; GCN-O2-NEXT: GCN NSA Reassign
; GCN-O2-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
+; GCN-O2-NEXT: AMDGPU Optimize VGPR Encoding
; GCN-O2-NEXT: Virtual Register Rewriter
; GCN-O2-NEXT: AMDGPU Mark Last Scratch Load
; GCN-O2-NEXT: Stack Slot Coloring
@@ -1428,9 +1431,9 @@
; GCN-O3-NEXT: Virtual Register Map
; GCN-O3-NEXT: Live Register Matrix
; GCN-O3-NEXT: Greedy Register Allocator
-; GCN-O3-NEXT: AMDGPU Optimize VGPR Encoding
; GCN-O3-NEXT: GCN NSA Reassign
; GCN-O3-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
+; GCN-O3-NEXT: AMDGPU Optimize VGPR Encoding
; GCN-O3-NEXT: Virtual Register Rewriter
; GCN-O3-NEXT: AMDGPU Mark Last Scratch Load
; GCN-O3-NEXT: Stack Slot Coloring
diff --git a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
index 1cc640a947a4f5..f33e5b614a9402 100644
--- a/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
+++ b/llvm/test/CodeGen/AMDGPU/sgpr-regalloc-flags.ll
@@ -33,6 +33,7 @@
; DEFAULT-NEXT: Greedy Register Allocator
; DEFAULT-NEXT: GCN NSA Reassign
; DEFAULT-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
+; DEFAULT-NEXT: AMDGPU Optimize VGPR Encoding
; DEFAULT-NEXT: Virtual Register Rewriter
; DEFAULT-NEXT: AMDGPU Mark Last Scratch Load
; DEFAULT-NEXT: Stack Slot Coloring
@@ -84,6 +85,7 @@
; BASIC-DEFAULT-NEXT: Greedy Register Allocator
; BASIC-DEFAULT-NEXT: GCN NSA Reassign
; BASIC-DEFAULT-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
+; BASIC-DEFAULT-NEXT: AMDGPU Optimize VGPR Encoding
; BASIC-DEFAULT-NEXT: Virtual Register Rewriter
; BASIC-DEFAULT-NEXT: AMDGPU Mark Last Scratch Load
; BASIC-DEFAULT-NEXT: Stack Slot Coloring
@@ -108,6 +110,7 @@
; DEFAULT-BASIC-NEXT: Basic Register Allocator
; DEFAULT-BASIC-NEXT: GCN NSA Reassign
; DEFAULT-BASIC-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
+; DEFAULT-BASIC-NEXT: AMDGPU Optimize VGPR Encoding
; DEFAULT-BASIC-NEXT: Virtual Register Rewriter
; DEFAULT-BASIC-NEXT: AMDGPU Mark Last Scratch Load
; DEFAULT-BASIC-NEXT: Stack Slot Coloring
@@ -139,6 +142,7 @@
; BASIC-BASIC-NEXT: Basic Register Allocator
; BASIC-BASIC-NEXT: GCN NSA Reassign
; BASIC-BASIC-NEXT: AMDGPU Rewrite AGPR-Copy-MFMA
+; BASIC-BASIC-NEXT: AMDGPU Optimize VGPR Encoding
; BASIC-BASIC-NEXT: Virtual Register Rewriter
; BASIC-BASIC-NEXT: AMDGPU Mark Last Scratch Load
; BASIC-BASIC-NEXT: Stack Slot Coloring
diff --git a/llvm/unittests/Target/AMDGPU/AMDGPUOptimizeVGPREncodingTest.cpp b/llvm/unittests/Target/AMDGPU/AMDGPUOptimizeVGPREncodingTest.cpp
new file mode 100644
index 00000000000000..185e019b5bb5e9
--- /dev/null
+++ b/llvm/unittests/Target/AMDGPU/AMDGPUOptimizeVGPREncodingTest.cpp
@@ -0,0 +1,407 @@
+//===- AMDGPUOptimizeVGPREncodingTest.cpp -----------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUOptimizeVGPREncoding.h"
+#include "AMDGPUUnitTests.h"
+#include "GCNRegPressure.h"
+#include "SIRegisterInfo.h"
+#include "llvm/CodeGen/LiveIntervals.h"
+#include "llvm/CodeGen/LiveRegMatrix.h"
+#include "llvm/CodeGen/MIRParser/MIRParser.h"
+#include "llvm/CodeGen/MachineFunctionAnalysis.h"
+#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/CodeGen/MachineScheduler.h"
+#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/CodeGen/TargetLowering.h"
+#include "llvm/Passes/PassBuilder.h"
+#include "llvm/Support/MathExtras.h"
+#include "gtest/gtest.h"
+
+using namespace llvm;
+
+/// MSB group, identified by an unsigned ID in [0, NumMSBGroups).
+using MSBGroup = unsigned;
+static constexpr unsigned MSBGroupSize = 256;
+static constexpr unsigned NumMSBGroups = 4;
+static constexpr unsigned DefaultGroup = 0;
+
+/// Operand type where the MSB group is relevant, identified by an unsigned ID
+/// in [0, NumOprdTypes).
+using OprdType = unsigned;
+static constexpr unsigned NumOprdTypes = 4;
+
+/// A VGPR definition with an initial physical mapping.
+struct VGPRDef {
+ /// The register's class, in MIR-spelling (e.g., "vgpr_32").
+ StringRef RegClass;
+ /// Index of the physical register, offset within the MSB group to the first
+ /// non-reserved VGPR.
+ unsigned PhysRegIdx;
+ /// MSB group to which the physical register must belong.
+ MSBGroup Group;
+};
+
+/// An "abstract" MODE-using instruction. We don't care about their exact nature
+/// for the sake of unit tests, just that they have specific operand sets that
+/// get their MSBs from MODE.
+///
+/// Instructions are created via static methods each mapping to different
+/// opcodes and operand-usage. All arguments to these functions must be
+/// registers (virtual or physical) in MIR-spelling (e.g.,
+/// "%0"/"$vgpr2_vgpr3"). Virtual registers used in source positions must have
+/// been defined through a \ref VGPRDef or a previous \p ModeUsingInstr in the
+/// instruction order. Physical registers may be reserved ones.
+struct ModeUsingInstr {
+ /// Creates a V_FMA_F32_e64 instruction.
+ static ModeUsingInstr vFMA(StringRef Dst, StringRef Src0, StringRef Src1,
+ StringRef Src2) {
+ ModeUsingInstr Instr(InstrType::V_FMA);
+ Instr.Regs[SRC0] = Src0;
+ Instr.Regs[SRC1] = Src1;
+ Instr.Regs[SRC2] = Src2;
+ Instr.Regs[DST] = Dst;
+ return Instr;
+ }
+
+ /// Serializes the instruction to string.
+ std::string toString() const {
+ SmallString<256> S;
+ switch (Ty) {
+ case InstrType::V_FMA:
+ return (Twine(Regs[DST]) + ":vgpr_32 = V_FMA_F32_e64 0, " +
+ Twine(Regs[SRC0]) + ":vgpr_32, 0, " + Twine(Regs[SRC1]) +
+ ":vgpr_32, 0, " + Twine(Regs[SRC2]) +
+ ":vgpr_32, 0, 0, implicit $mode, implicit $exec")
+ .toNullTerminatedStringRef(S)
+ .str();
+ }
+ }
+
+private:
+ enum { SRC0 = 0, SRC1 = 1, SRC2 = 2, DST = 3 };
+ std::array<StringRef, NumOprdTypes> Regs;
+
+ enum class InstrType { V_FMA };
+ InstrType Ty;
+
+ ModeUsingInstr(InstrType Ty) : Ty(Ty) { Regs.fill(""); }
+};
+
+class AMDGPUOptimizeVGPREncodingTest : public AMDGPUCodeGenTestBase {
+public:
+ MachineFunction *MF;
+ VirtRegMap *VRM;
+ LiveRegMatrix *LRM;
+
+ MachineRegisterInfo *MRI;
+ const LiveIntervals *LIS;
+ const SIRegisterInfo *TRI;
+ const SIInstrInfo *TII;
+ RegisterClassInfo RegClassInfo;
+ unsigned NumFreeVGPRsPerGroup;
+
+ void SetUp() override { setUpImpl("amdgpu12.50--", "", ""); }
+
+ /// Marks every VGPR as reserved except the top NumFreeVGPRsPerGroup registers
+ /// of each MSB group, so that the pass only ever has that small window in
+ /// each MSB group to (re-)assign registers into.
+ void reserveVGPRs() {
+ const TargetRegisterClass &RC = AMDGPU::VGPR_32RegClass;
+ for (unsigned I = 0, E = RC.getNumRegs(); I != E; ++I) {
+ if (I % MSBGroupSize < MSBGroupSize - NumFreeVGPRsPerGroup)
+ MRI->reserveReg(RC.getRegister(I), TRI);
+ }
+ }
+
+ /// Programatically creates an MIR function and returns whether it
+ /// successfully parsed. The function's entry block contains an IMPLICIT_DEF
+ /// for all VGPRs in \p RegDefs, which are assigned to starting physical
+ /// registers. Then, as many blocks are created as elements in \p MIRBlocks,
+ /// and each is populated with each array element's list of MODE-using
+ /// instructions. Finally, an exit block adds an implicit use for all VGPRs in
+ /// \p RegDefs, ensuring they are live over all middle blocks. To constrain
+ /// register re-assignments and make tests more trackable, all VGPRs but the
+ /// top \p NumFreeVGPRsPerGroup in each MSB group are marked reserved.
+ bool createMIRAndAssign(
+ ArrayRef<VGPRDef> RegDefs,
+ ArrayRef<const SmallVectorImpl<ModeUsingInstr> *> MIRBlocks,
+ unsigned NumFreeVGPRsPerGroup) {
+ assert(NumFreeVGPRsPerGroup < MSBGroupSize);
+
+ std::string MIRString = R"MIR(
+--- |
+ define amdgpu_kernel void @func() #0 {
+ ret void
+ }
+ attributes #0 = { "amdgpu-flat-work-group-size"="1,32" }
+...
+---
+name: func
+tracksRegLiveness: true
+machineFunctionInfo:
+ isEntryFunction: true
+body: |
+ bb.0:
+)MIR";
+
+ // All registrer definitions go in the entry block. All MODE-using
+ // instructions go in subsequent blocks, and finally an exit block with
+ // implicit uses of all registers to extend their live-range over the entire
+ // interesting part of the function.
+ std::string RegisterUses;
+ for (const auto &[VirtRegIdx, Assignment] : enumerate(RegDefs)) {
+ // Each virtual register gets an IMPLICIT_DEF.
+ MIRString += " %" + std::to_string(VirtRegIdx) + ':' +
+ Assignment.RegClass.str() + " = IMPLICIT_DEF\n";
+ // Accumulate uses for later.
+ RegisterUses += ", implicit %" + std::to_string(VirtRegIdx);
+ }
+ for (const auto &[Idx, Block] : enumerate(MIRBlocks)) {
+ MIRString += " bb." + std::to_string(Idx + 1) + ":\n";
+ for (const ModeUsingInstr &Instr : *Block)
+ MIRString += " " + Instr.toString() + '\n';
+ }
+ MIRString += " bb." + std::to_string(MIRBlocks.size() + 1) +
+ ":\n S_NOP 0" + RegisterUses + "\n...\n";
+ if (!parseMIR(MIRString))
+ return false;
+
+ MF = &getMF("func");
+ VRM = &MFAM.getResult<VirtRegMapAnalysis>(*MF);
+ LRM = &MFAM.getResult<LiveRegMatrixAnalysis>(*MF);
+ MRI = &MF->getRegInfo();
+ LIS = &MFAM.getResult<LiveIntervalsAnalysis>(*MF);
+ TRI = static_cast<const SIRegisterInfo *>(&VRM->getTargetRegInfo());
+ TII =
+ static_cast<const GCNSubtarget *>(&MF->getSubtarget())->getInstrInfo();
+ RegClassInfo.runOnMachineFunction(*MF);
+
+ // Reserve VGPRs except the last NumFreeVGPRsPerGroup in each group, then
+ // create an initial virtual-to-physical assignment using free physical
+ // registers.
+ this->NumFreeVGPRsPerGroup = NumFreeVGPRsPerGroup;
+ reserveVGPRs();
+ for (const auto &[VirtRegIdx, Assignment] : enumerate(RegDefs))
+ assign(VirtRegIdx, Assignment.PhysRegIdx, Assignment.Group);
+ return true;
+ }
+
+ /// Assigns virtual register with index \p VirtRegIdx to physical
+ /// register \p PhysRegIdx (offset within the MSB group to the first
+ /// non-reserved VGPR) in \p Group.
+ void assign(unsigned VirtRegIdx, unsigned PhysRegIdx, MSBGroup Group) {
+ assert(Group < NumMSBGroups && "invalid MSB group");
+
+ Register VirtReg = Register::index2VirtReg(VirtRegIdx);
+ assert(LIS->hasInterval(VirtReg) && "invalid virt index");
+ const LiveInterval &LI = LIS->getInterval(VirtReg);
+ const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);
+ unsigned Width = divideCeil(TRI->getRegSizeInBits(RC), 32);
+ assert(PhysRegIdx < NumFreeVGPRsPerGroup / Width && "invalid phys idx");
+
+ // PhysRegIdx selects one of the Width-sized slots of the free window at
+ // the top of the MSB group.
+ MCRegister Lo = AMDGPU::VGPR_32RegClass.getRegister(
+ Group * MSBGroupSize + MSBGroupSize - NumFreeVGPRsPerGroup +
+ PhysRegIdx * Width);
+ MCRegister PhysReg =
+ Width == 1 ? Lo : TRI->getMatchingSuperReg(Lo, AMDGPU::sub0, &RC);
+ assert(PhysReg && !MRI->isReserved(PhysReg) && "register must be free");
+ LRM->assign(LI, PhysReg);
+ }
+
+ /// Returns \p Reg's MSB group.
+ MSBGroup getVGPRGroup(Register Reg) const {
+ MCRegister PhysReg = Reg.isVirtual() ? VRM->getPhys(Reg) : Reg.asMCReg();
+ return TRI->getHWRegIndex(PhysReg) >> 8;
+ }
+
+ /// Counts the number of S_SET_VGPR_MSB instructions the \ref MF will require.
+ unsigned countSetModeInstrs() const {
+ unsigned NumSetModeInstrs = 0;
+ for (const MachineBasicBlock &MBB : *MF)
+ NumSetModeInstrs += countSetModeInstrs(MBB);
+ return NumSetModeInstrs;
+ }
+
+ /// Counts the number of S_SET_VGPR_MSB instructions \p MBB will require.
+ unsigned countSetModeInstrs(const MachineBasicBlock &MBB) const {
+ unsigned NumSetModeInstrs = 0;
+
+ // A std::nullopt for a particular operand type means that there exists a
+ // previous S_SET_VGPR_MSB instruction whose group for that operand type is
+ // not yet constrained i.e., onto which we can piggyback a later group
+ // requirement.
+ std::array<std::optional<MSBGroup>, NumOprdTypes> CurrentGroups, MIGroups;
+
+ // Groups start with all MSBs set to the default group.
+ CurrentGroups.fill(DefaultGroup);
+
+ for (const MachineInstr &MI : MBB) {
+ const MCInstrDesc &Desc = MI.getDesc();
+ const auto [Table, VOPDTable] =
+ AMDGPU::getVGPRLoweringOperandTables(Desc);
+ if (!Table)
+ continue;
+
+ auto GetRelevantMO =
+ [&](const AMDGPU::OpName &Name) -> const MachineOperand * {
+ if (Name == AMDGPU::OpName::NUM_OPERAND_NAMES)
+ return nullptr;
+
+ const MachineOperand *MO = TII->getNamedOperand(MI, Name);
+ if (!MO || !MO->isReg())
+ return nullptr;
+
+ Register Reg = MO->getReg();
+ const TargetRegisterClass *RC = TRI->getRegClassForReg(*MRI, Reg);
+ return (RC && SIRegisterInfo::isVGPRClass(RC)) ? MO : nullptr;
+ };
+
+ MIGroups.fill(std::nullopt);
+ for (OprdType Oprd : seq(NumOprdTypes)) {
+ const MachineOperand *MO = GetRelevantMO(Table[Oprd]);
+ if (!MO)
+ continue;
+
+ // Tied src2 uses of VOP2 and 32-bit-encoded VOP3 only depend on the
+ // vdst bit and are handled with the def, so they are not their own
+ // operand for MSB group purposes.
+ if (Table[Oprd] == AMDGPU::OpName::src2 && !MO->isDef() &&
+ MO->isTied() &&
+ (SIInstrInfo::isVOP2(MI) ||
+ (SIInstrInfo::isVOP3(MI) &&
+ TII->hasVALU32BitEncoding(MI.getOpcode()))))
+ continue;
+
+ MIGroups[Oprd] = getVGPRGroup(MO->getReg());
+ }
+
+ if (VOPDTable) {
+ for (OprdType Oprd : seq(NumOprdTypes)) {
+ const MachineOperand *MO = GetRelevantMO(VOPDTable[Oprd]);
+ if (MO)
+ MIGroups[Oprd] = getVGPRGroup(MO->getReg());
+ }
+ }
+
+ // Merge group requirements from the MI with the current ones, and
+ // determine whether we need a new S_SET_VGPR_MSB.
+ for (auto [Current, Requirement] : zip(CurrentGroups, MIGroups)) {
+ if (Current.has_value()) {
+ if (Requirement.has_value() && *Current != *Requirement) {
+ ++NumSetModeInstrs;
+ CurrentGroups = MIGroups;
+ break;
+ }
+ } else {
+ // Piggyback into previous S_SET_VGPR_MSB instruction.
+ Current = Requirement;
+ }
+ }
+ }
+
+ // Groups must end with all MSBs set to the default group so an extra
+ // S_SET_VGPR_MSB instruction would be inserted if that is not the case.
+ for (const std::optional<MSBGroup> &Group : CurrentGroups) {
+ if (Group.has_value() && *Group != DefaultGroup) {
+ ++NumSetModeInstrs;
+ break;
+ }
+ }
+
+ return NumSetModeInstrs;
+ }
+
+ /// Runs the optimization pass, expecting \p ExpectModeSetBefore
+ /// S_SET_VGPR_MSB instructions to be required in the function before it, and
+ /// \p ExpectModeSetAfter after it.
+ void runWithExpectation(unsigned ExpectModeSetBefore,
+ unsigned ExpectModeSetAfter) {
+ unsigned ActualModeSetBefore = countSetModeInstrs();
+ EXPECT_EQ(ExpectModeSetBefore, ActualModeSetBefore);
+
+ AMDGPUOptimizeVGPREncodingPass Pass;
+ Pass.run(*MF, MFAM);
+
+ unsigned ActualModeSetAfter = countSetModeInstrs();
+ EXPECT_EQ(ExpectModeSetAfter, ActualModeSetAfter);
+ }
+};
+
+/// All registers of the VFMA are in group 1, requiring S_SET_VGPR_MSB around
+/// the instruction because all operand types start (and must end) in the
+/// default group.
+TEST_F(AMDGPUOptimizeVGPREncodingTest, ReassignToDefaultGroup) {
+ SmallVector<VGPRDef> Registers{
+ {"vgpr_32", 0, 1}, // %0
+ {"vgpr_32", 1, 1}, // %1
+ {"vgpr_32", 2, 1}, // %2
+ {"vgpr_32", 3, 1}, // %3
+ };
+ SmallVector<ModeUsingInstr> Instructions{
+ ModeUsingInstr::vFMA("%0", "%1", "%2", "%3")};
+
+ ASSERT_TRUE(createMIRAndAssign(Registers, {&Instructions}, 8));
+ runWithExpectation(2, 0);
+}
+
+/// Each VFMA has all its registers in the same non-default group, and the two
+/// VFMAs have different groups. All registers should all be re-assigned to the
+/// default group which has just enough free registers. The pass must avoid a
+/// local-maxima where all registers end up in one of the VFMA's group.
+TEST_F(AMDGPUOptimizeVGPREncodingTest, ReassignToDefaultGroupVFMAConflict) {
+ SmallVector<VGPRDef> Registers{
+ {"vgpr_32", 0, 1}, // %0
+ {"vgpr_32", 1, 1}, // %1
+ {"vgpr_32", 2, 1}, // %2
+ {"vgpr_32", 3, 1}, // %3
+ {"vgpr_32", 0, 2}, // %4
+ {"vgpr_32", 1, 2}, // %5
+ {"vgpr_32", 2, 2}, // %6
+ {"vgpr_32", 3, 2}, // %7
+ };
+ SmallVector<ModeUsingInstr> Instructions{
+ ModeUsingInstr::vFMA("%0", "%1", "%2", "%3"),
+ ModeUsingInstr::vFMA("%4", "%5", "%6", "%7")};
+ ASSERT_TRUE(createMIRAndAssign(Registers, {&Instructions}, 8));
+ runWithExpectation(3, 0);
+}
+
+/// Each VFMA has each of its register in a different group, and consecutive
+/// VFMAs have matching operands in different groups. There is a single free
+/// register in each group, which makes it hard for the pass to find
+/// re-assignments, even though there is a solution that only requires a single
+/// S_SET_VGPR_MSB.
+TEST_F(AMDGPUOptimizeVGPREncodingTest, SingleFreeRegPerGroup) {
+ SmallVector<VGPRDef> Registers{
+ {"vgpr_32", 0, 0}, // %0
+ {"vgpr_32", 0, 1}, // %1
+ {"vgpr_32", 0, 2}, // %2
+ {"vgpr_32", 0, 3}, // %3
+ {"vgpr_32", 1, 1}, // %4
+ {"vgpr_32", 1, 2}, // %5
+ {"vgpr_32", 1, 3}, // %6
+ {"vgpr_32", 1, 0}, // %7
+ {"vgpr_32", 2, 2}, // %8
+ {"vgpr_32", 2, 3}, // %9
+ {"vgpr_32", 2, 0}, // %10
+ {"vgpr_32", 2, 1}, // %11
+ {"vgpr_32", 3, 3}, // %12
+ {"vgpr_32", 3, 0}, // %13
+ {"vgpr_32", 3, 1}, // %14
+ {"vgpr_32", 3, 2}, // %15
+ };
+ SmallVector<ModeUsingInstr> Instructions{
+ ModeUsingInstr::vFMA("%0", "%1", "%2", "%3"),
+ ModeUsingInstr::vFMA("%4", "%5", "%6", "%7"),
+ ModeUsingInstr::vFMA("%8", "%9", "%10", "%11"),
+ ModeUsingInstr::vFMA("%12", "%13", "%14", "%15")};
+ ASSERT_TRUE(createMIRAndAssign(Registers, {&Instructions}, 5));
+ runWithExpectation(5, 5);
+}
diff --git a/llvm/unittests/Target/AMDGPU/CMakeLists.txt b/llvm/unittests/Target/AMDGPU/CMakeLists.txt
index 2bb9b6dedba13e..2681ebb533ba2f 100644
--- a/llvm/unittests/Target/AMDGPU/CMakeLists.txt
+++ b/llvm/unittests/Target/AMDGPU/CMakeLists.txt
@@ -24,6 +24,7 @@ set(LLVM_LINK_COMPONENTS
add_llvm_target_unittest(AMDGPUTests
AMDGPUMCExprTest.cpp
+ AMDGPUOptimizeVGPREncodingTest.cpp
AMDGPUUnitTests.cpp
CSETest.cpp
DwarfRegMappings.cpp
>From 04c923a480a663b86c2465e1d446f2e9ef832aa5 Mon Sep 17 00:00:00 2001
From: Lucas Ramirez <lucas.rami at proton.me>
Date: Tue, 29 Sep 2026 12:45:36 +0000
Subject: [PATCH 4/5] More tests and some cleanup/bugfixes
Notably, fixes incorrect conflict resolution logic with physical
registers and typo in `OptRegCandidate::operator<` that created an
incorrect candidate order.
---
.../AMDGPU/AMDGPUOptimizeVGPREncoding.cpp | 295 +++++-----
.../AMDGPU/AMDGPUOptimizeVGPREncoding.h | 2 +-
llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll | 2 +
.../AMDGPU/AMDGPUOptimizeVGPREncodingTest.cpp | 538 ++++++++++++++----
4 files changed, 568 insertions(+), 269 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
index 94929dba566c41..5111c1a0dc0f79 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
@@ -61,6 +61,7 @@ static constexpr unsigned DefaultGroup = 0;
/// Groups are 256 registers each.
static constexpr unsigned GroupSizeLog2 = 8;
+static constexpr unsigned GroupSize = 1 << GroupSizeLog2;
/// Operand type where the MSB group is relevant, identified by an unsigned ID
/// in [0, NumOprdTypes).
@@ -120,20 +121,6 @@ class MBBModeUsage {
return Instructions[Idx];
}
- /// Returns the index of the first instruction with a MODE-reading operand of
- /// type \p Oprd in the block, or \ref ModeInstr::NoIdx if none exists.
- unsigned getFirstOprd(OprdType Oprd) const {
- return Instructions.empty() ? ModeInstr::NoIdx : getFirstInstrFrom(0, Oprd);
- }
-
- /// Returns the index of the last instruction with a MODE-reading operand of
- /// type \p Oprd in the block, or \ref ModeInstr::NoIdx if none exists.
- unsigned getLastOprd(OprdType Oprd) const {
- return Instructions.empty()
- ? ModeInstr::NoIdx
- : getLastInstrUntil(Instructions.size() - 1, Oprd);
- }
-
/// Returns the index of the first instruction from \p InstrIdx (included)
/// with a MODE-reading operand of type \p Oprd in the block, or \ref
/// ModeInstr::NoIdx if none exists.
@@ -162,7 +149,7 @@ class MBBModeUsage {
return getInstrIdxImpl<true, true>(InstrIdx, Oprd);
}
-#ifndef NDEBUG
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Printable print(const VirtRegMap &VRM) const;
#endif
@@ -298,7 +285,7 @@ class OptReg {
/// is illegal to change the MSB group of a pinned register.
void notifyPhysAssignmentChanged(MSBGroup NewGroup);
-#ifndef NDEBUG
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Printable print(const VirtRegMap &VRM) const;
#endif
@@ -412,7 +399,7 @@ class ModeSetOptimizer {
public:
/// Initializes the optimizer with all optimizable registers in \p OptRegs and
/// all blocks in \p ModeUsage. Performs a first round of register pinning and
- /// conflict resolution on all relevant registers.
+ /// conflict resolution on all registers.
ModeSetOptimizer(OptimizableRegs &OptRegs, ArrayRef<MBBModeUsage> ModeUsage,
const VirtRegMap &VRM);
@@ -451,6 +438,14 @@ class ModeSetOptimizer {
ArrayRef<MBBModeUsage> ModeUsage;
const VirtRegMap &VRM;
+ /// Around an occurence of \p Reg at \p Position, resolves any conflict with a
+ /// neighbor pinned outside MSB group \p PreferredGroup. \p PrevNeighbor
+ /// determines whether the method looks at the neighbor before or after the
+ /// position.
+ template <bool PrevNeighbor>
+ void resolveConflictWithNeighbor(OptReg &Reg, MSBGroup PreferredGroup,
+ const OptReg::Coordinates &Position);
+
/// Resolves conflicts for \p Reg, if any, and returns whether the register
/// had conflicts.
bool resolveConflictingPins(OptReg &Reg);
@@ -552,11 +547,11 @@ class OptRegCandidate {
/// Creates the candidate for \p Reg.
OptRegCandidate(OptReg &Reg) : Reg(Reg), Targets(NumMSBGroups) {
assert(!Reg.isPinned() && "register cannot be pinned initially");
- update();
+ recomputeTargets();
}
/// Re-computes the candidate's target groups and potential benefit.
- void update();
+ void recomputeTargets();
/// Whether the candidate has a profitable re-assignment to any MSB group
/// i.e., a re-assigment that will strictly increase the combined score of all
@@ -574,7 +569,7 @@ class OptRegCandidate {
bool operator<(const OptRegCandidate &Other) const;
-#ifndef NDEBUG
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Printable print(const VirtRegMap &VRM) const;
#endif
@@ -606,7 +601,7 @@ class MaxHeap {
/// Returns whether the heap is empty.
bool empty() const { return HeapToSlot.empty(); }
-#ifndef NDEBUG
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Printable print(const VirtRegMap &VRM, const LiveIntervals &LIS) const;
#endif
@@ -649,15 +644,15 @@ class VirtRegReMap {
VirtRegReMap(LiveRegMatrix &LRM, LiveIntervals &LIS,
const MachineFunction &MF, const VirtRegMap &VRM);
- /// Attempts to find an available physical register in MSB group \p Dst that
- /// virtual register \p Reg can be assigned to. Returns the first such
- /// register it finds, or the sentinel register if none could be found.
- MCRegister tryAssignInGroup(Register Reg, MSBGroup Group);
-
/// Attempts to re-assign virtual register \p Reg to an available physical
- /// register in any of the MSB groups indicated by set bits in \p Targets.
- /// Returns whether any re-assignment took place.
- bool tryAssignToTargetGroups(OptReg &Reg, const SmallBitVector &Targets);
+ /// register in any of the MSB groups indicated by set bits in \p
+ /// TargetGroups. Returns whether any re-assignment took place.
+ bool tryReAssign(OptReg &Reg, const SmallBitVector &TargetGroups);
+
+ /// Determines whether register \p PhysReg in class \p RC crosses an MSB group
+ /// boundary.
+ bool regCrossesGroupBoundary(MCRegister PhysReg,
+ const TargetRegisterClass &RC) const;
private:
LiveRegMatrix &LRM;
@@ -665,7 +660,6 @@ class VirtRegReMap {
RegisterClassInfo RCI;
const VirtRegMap &VRM;
const SIRegisterInfo &TRI;
- const BitVector &ReservedRegs;
};
class AMDGPUOptimizeVGPREncoding {
@@ -684,7 +678,7 @@ class AMDGPUOptimizeVGPREncoding {
} // namespace
-#ifndef NDEBUG
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
static std::string printGroup(MSBGroup Group) {
return "MSB#" + std::to_string(Group);
}
@@ -961,7 +955,7 @@ OptimizableRegs::OptimizableRegs(const BitVector &OptVirtRegs,
for (OprdType Oprd : seq(NumOprdTypes)) {
OptReg *PreviousOptReg = nullptr;
- unsigned InstrIdx = BlockUsage.getFirstOprd(Oprd);
+ unsigned InstrIdx = BlockUsage.getFirstInstrFrom(0, Oprd);
while (InstrIdx != ModeInstr::NoIdx) {
const ModeInstr &CurrentInstr = Instructions[InstrIdx];
@@ -1016,19 +1010,12 @@ ModeSetOptimizer::ModeSetOptimizer(OptimizableRegs &OptRegs,
ArrayRef<MBBModeUsage> ModeUsage,
const VirtRegMap &VRM)
: OptRegs(OptRegs), CheckShouldBePinned(OptRegs.getAllOptVirtRegs()),
- CheckResolveConflict(OptRegs.getNumVirtRegs()),
+ CheckResolveConflict(OptRegs.getAllOptVirtRegs()),
ModeSetPlacement(ModeUsage.size()), ModeUsage(ModeUsage), VRM(VRM) {
for (const auto &[MBB, Placement] : zip_equal(ModeUsage, ModeSetPlacement))
Placement.resize(MBB.getInstructions().size() + 1);
- // We initially check all registers for pin-eligibility, and those that are
- // neighbors of already pinned registers for conflicting pins.
- for (const OptReg &Reg : OptRegs) {
- if (!Reg.isPinned())
- continue;
- for (const auto &[NeighborReg, _] : Reg.getNeighbors())
- CheckResolveConflict.set(NeighborReg->getVirtIndex());
- }
+ // We initially check all registers for conflicts and pin-eligibility.
resolveConflictsAndPinRegs();
}
@@ -1136,6 +1123,67 @@ void ModeSetOptimizer::breakNeighborRelationship(unsigned MBBIdx,
}
}
+template <bool PrevNeighbor>
+void ModeSetOptimizer::resolveConflictWithNeighbor(
+ OptReg &Reg, MSBGroup PreferredGroup, const OptReg::Coordinates &Position) {
+ const auto &[MBBIdx, InstrIdx, Oprd] = Position;
+ const MBBModeUsage &MBB = ModeUsage[MBBIdx];
+
+ // Get the neighbor's index.
+ unsigned NeighborIdx;
+ if constexpr (PrevNeighbor)
+ NeighborIdx = MBB.getLastInstrBefore(InstrIdx, Oprd);
+ else
+ NeighborIdx = MBB.getFirstInstrAfter(InstrIdx, Oprd);
+
+ // We place MODE-setting instructions between us and conflicting neighbors.
+ constexpr auto PlaceAround = PrevNeighbor ? &ModeSetOptimizer::placeJustBefore
+ : &ModeSetOptimizer::placeJustAfter;
+
+ if (NeighborIdx == ModeInstr::NoIdx) {
+ // This indicates that we neighbor the block's boundary which generates a
+ // pin to the default MSB group.
+ if (PreferredGroup == DefaultGroup)
+ return;
+
+ // A MODE-set between the register and the block boundary avoids a conflict.
+ constexpr auto HasModeSetBetween = PrevNeighbor
+ ? &ModeSetOptimizer::hasModeSetBefore
+ : &ModeSetOptimizer::hasModeSetAfter;
+ if ((this->*HasModeSetBetween)(MBBIdx, InstrIdx))
+ return;
+
+ LLVM_DEBUG(dbgs() << " Resolving default pin " << printOprdType(Oprd)
+ << " in MBB#" << MBB.MBB.getNumber() << '\n');
+ (this->*PlaceAround)(MBBIdx, InstrIdx);
+ return;
+ }
+
+ // A MODE-set between the register and its neighbor avoids a conflict.
+ const auto &[AfterIdx, BeforeIdx] =
+ PrevNeighbor ? std::pair<unsigned, unsigned>{NeighborIdx, InstrIdx}
+ : std::pair<unsigned, unsigned>{InstrIdx, NeighborIdx};
+ if (hasModeSetBetween(MBBIdx, AfterIdx, BeforeIdx))
+ return;
+
+ // A neighbor register pinned in a non-preferred group constitutes a conflict.
+ // Physical registers are pinned by definition.
+ Register NeighborReg = MBB.getInstructions()[NeighborIdx].Oprds[Oprd];
+ MSBGroup NeighborGroup = getVGPRGroup(NeighborReg, VRM);
+ if (PreferredGroup == NeighborGroup)
+ return;
+ OptReg *NeighborOptReg = OptRegs[NeighborReg];
+ if (NeighborOptReg && !NeighborOptReg->isPinned())
+ return;
+
+ LLVM_DEBUG(dbgs() << " Resolving conflict with neighbor "
+ << printReg(NeighborReg, &VRM.getTargetRegInfo(), 0,
+ &VRM.getRegInfo())
+ << " for operand type " << printOprdType(Oprd) << " in MBB#"
+ << MBB.MBB.getNumber() << '\n');
+ (this->*PlaceAround)(MBBIdx, InstrIdx);
+}
+
bool ModeSetOptimizer::resolveConflictingPins(OptReg &Reg) {
if (Reg.getPinGroups().count() <= 1)
return false;
@@ -1145,78 +1193,14 @@ bool ModeSetOptimizer::resolveConflictingPins(OptReg &Reg) {
// Only one MSB group with pinned neighbors must remain.
MSBGroup PreferredGroup = selectPreferredMSBGroup(Reg);
assert(PreferredGroup < NumMSBGroups && "invalid group");
-
LLVM_DEBUG(dbgs() << " Preferred group is " << printGroup(PreferredGroup)
<< '\n');
- // Conflicts with block boundary pins may need to be resolved when the default
- // group is not the preferred one.
- if (PreferredGroup != DefaultGroup &&
- Reg.getGroupPinnedScore(DefaultGroup) > 0) {
- for (const auto &[MBBIdx, InstrIdx, Oprd] : Reg.getOccurrences()) {
- const MBBModeUsage &MBB = ModeUsage[MBBIdx];
- if (InstrIdx == MBB.getFirstOprd(Oprd) &&
- !hasModeSetBefore(MBBIdx, InstrIdx)) {
- LLVM_DEBUG(dbgs() << " Resolving default entry pin "
- << printOprdType(Oprd) << " in MBB#"
- << MBB.MBB.getNumber() << '\n');
- placeJustBefore(MBBIdx, InstrIdx);
- }
- if (InstrIdx == MBB.getLastOprd(Oprd) &&
- !hasModeSetAfter(MBBIdx, InstrIdx)) {
- LLVM_DEBUG(dbgs() << " Resolving default exit pin "
- << printOprdType(Oprd) << " in MBB#"
- << MBB.MBB.getNumber() << '\n');
- placeJustAfter(MBBIdx, InstrIdx);
- }
- }
- }
-
- // Conflicts with occurrences of pinned neighbors in the non-preferred MSB
- // group need to be resolved. Iterate over a copy of the list of neighbors
- // because they will be modified as we simulate placement of MODE-setting
- // instructions.
- OptReg::WeightedNeighbors Neighbors(Reg.getNeighbors());
- for (const auto &[NeighborReg, _] : Neighbors) {
- // Early exit when we know we are not going to find conflicting occurrences.
- if (!NeighborReg->isPinned() || NeighborReg->getMSB() == PreferredGroup ||
- Reg.getGroupPinnedScore(NeighborReg->getMSB()) == 0)
- continue;
-
- // Look through occurrences for conflicts.
- for (const auto &[MBBIdx, InstrIdx, Oprd] : NeighborReg->getOccurrences()) {
- const MBBModeUsage &MBB = ModeUsage[MBBIdx];
- ArrayRef<ModeInstr> Instructions = MBB.getInstructions();
-
- // Look at the operand immediately before this neighbor occurrence. If it
- // matches the register for which we are currently resolving conflicts,
- // then it is one of the neighborhood relationships to break.
- unsigned PrevIdx = MBB.getLastInstrBefore(InstrIdx, Oprd);
- if (PrevIdx != ModeInstr::NoIdx &&
- Instructions[PrevIdx].Oprds[Oprd] == Reg.getVirt() &&
- !hasModeSetBetween(MBBIdx, PrevIdx, InstrIdx)) {
-
- LLVM_DEBUG(dbgs() << " Resolving with next neighbor "
- << NeighborReg->print(VRM) << " for operand type "
- << printOprdType(Oprd) << " in MBB#"
- << MBB.MBB.getNumber() << '\n');
- placeJustBefore(MBBIdx, InstrIdx);
- }
-
- // Look at the operand immediately after this neighbor occurrence. If it
- // matches the register for which we are currently resolving conflicts,
- // then it is one of the neighborhood relationships to break.
- unsigned NextIdx = MBB.getFirstInstrAfter(InstrIdx, Oprd);
- if (NextIdx != ModeInstr::NoIdx &&
- Instructions[NextIdx].Oprds[Oprd] == Reg.getVirt() &&
- !hasModeSetBetween(MBBIdx, InstrIdx, NextIdx)) {
- LLVM_DEBUG(dbgs() << " Resolving with previous neighbor "
- << NeighborReg->print(VRM) << " for operand type "
- << printOprdType(Oprd) << " in MBB#"
- << MBB.MBB.getNumber() << '\n');
- placeJustBefore(MBBIdx, NextIdx);
- }
- }
+ for (const OptReg::Coordinates &Position : Reg.getOccurrences()) {
+ resolveConflictWithNeighbor</*IsNeighborBefore=*/true>(Reg, PreferredGroup,
+ Position);
+ resolveConflictWithNeighbor</*IsNeighborBefore=*/false>(Reg, PreferredGroup,
+ Position);
}
LLVM_DEBUG(dbgs() << " | Updated register: " << Reg.print(VRM) << '\n');
@@ -1312,14 +1296,14 @@ bool OptRegCandidate::operator<(const OptRegCandidate &Other) const {
return NeighboringPins < Other.NeighboringPins;
// Higher benefit wins.
- if (Benefit < Other.Benefit)
+ if (Benefit != Other.Benefit)
return Benefit < Other.Benefit;
// Break ties with unique virtual register index.
return Reg.getVirtIndex() < Other.Reg.getVirtIndex();
}
-void OptRegCandidate::update() {
+void OptRegCandidate::recomputeTargets() {
Targets.reset();
if (Reg.isPinned())
return;
@@ -1402,7 +1386,7 @@ void MaxHeap::reorderIfExists(const OptReg &Reg) {
if (Cand == OptRegToSlotIdx.end())
return;
Slot &S = Slots[Cand->second];
- S.Cand.update();
+ S.Cand.recomputeTargets();
// Re-order the tree around the updated slot.
if (!siftUp(S.HeapIdx))
@@ -1452,54 +1436,59 @@ bool MaxHeap::siftDown(unsigned HeapIdx) {
VirtRegReMap::VirtRegReMap(LiveRegMatrix &LRM, LiveIntervals &LIS,
const MachineFunction &MF, const VirtRegMap &VRM)
: LRM(LRM), LIS(LIS), VRM(VRM),
- TRI(*static_cast<const SIRegisterInfo *>(&VRM.getTargetRegInfo())),
- ReservedRegs(MF.getRegInfo().getReservedRegs()) {
+ TRI(*static_cast<const SIRegisterInfo *>(&VRM.getTargetRegInfo())) {
RCI.runOnMachineFunction(MF);
}
-static bool overflowsLastGroup(unsigned HWRegIdx, unsigned NumLanes,
- MSBGroup Group) {
- if (Group != NumMSBGroups - 1)
- return false;
- return ((HWRegIdx + NumLanes - 1) >> GroupSizeLog2) != NumMSBGroups - 1;
-}
+bool VirtRegReMap::tryReAssign(OptReg &Reg,
+ const SmallBitVector &TargetGroups) {
+ assert(!TargetGroups.test(Reg.getMSB()) && "target is current MSB group");
-MCRegister VirtRegReMap::tryAssignInGroup(Register VirtReg, MSBGroup Group) {
- assert(VirtReg.isVirtual() && "expected virtreg");
- const LiveInterval &RegLI = LIS.getInterval(VirtReg);
+ Register VirtReg = Reg.getVirt();
+ const LiveInterval &LI = LIS.getInterval(VirtReg);
+ MCRegister OriginalPhys = VRM.getPhys(VirtReg);
const MachineRegisterInfo &MRI = VRM.getRegInfo();
- const TargetRegisterClass &RC = *MRI.getRegClass(VirtReg);
- const unsigned NumLanes = divideCeil(TRI.getRegSizeInBits(RC), 32);
- for (MCPhysReg CandPhysReg : RCI.getOrder(&RC)) {
- if (ReservedRegs[CandPhysReg] || getVGPRGroup(CandPhysReg, VRM) != Group ||
- overflowsLastGroup(TRI.getHWRegIndex(CandPhysReg), NumLanes, Group))
+ const TargetRegisterClass *RC = MRI.getRegClass(VirtReg);
+
+ // When the currently assigned physical register crosses into the next MSB
+ // group and that group is a target group, we have to unassign the original
+ // interval as to not cause spurious interferences with candidate physical
+ // registers at the beginning of the next group. This avoids
+ // unassigning/re-assigning the same physical register in most failed
+ // re-assignments.
+ const bool UnassignOriginalReg = regCrossesGroupBoundary(OriginalPhys, *RC) &&
+ TargetGroups.test(Reg.getMSB() + 1);
+ if (UnassignOriginalReg)
+ LRM.unassign(LI);
+
+ for (MCPhysReg CandPhysReg : RCI.getOrder(RC)) {
+ MSBGroup CandGroup = getVGPRGroup(CandPhysReg, VRM);
+ if (!TargetGroups.test(CandGroup) ||
+ LRM.checkInterference(LI, CandPhysReg) != LiveRegMatrix::IK_Free)
continue;
- if (LRM.checkInterference(RegLI, CandPhysReg) == LiveRegMatrix::IK_Free)
- return CandPhysReg;
+
+ if (!UnassignOriginalReg)
+ LRM.unassign(LI);
+ LRM.assign(LI, CandPhysReg);
+ Reg.notifyPhysAssignmentChanged(getVGPRGroup(CandPhysReg, VRM));
+ return true;
}
- return MCPhysReg();
+
+ // We failed to find a register.
+ if (UnassignOriginalReg)
+ LRM.assign(LI, OriginalPhys);
+ return false;
}
-bool VirtRegReMap::tryAssignToTargetGroups(OptReg &Reg,
- const SmallBitVector &Targets) {
- Register VirtReg = Reg.getVirt();
- const LiveInterval &LI = LIS.getInterval(VirtReg);
+bool VirtRegReMap::regCrossesGroupBoundary(
+ MCRegister PhysReg, const TargetRegisterClass &RC) const {
+ unsigned NumLanes = divideCeil(RC.getSizeInBits(), 32);
+ if (NumLanes == 1)
+ return false;
- MCRegister OriginalPhys = VRM.getPhys(VirtReg);
- LRM.unassign(LI);
-
- for (MSBGroup Target : Targets.set_bits()) {
- assert(Target != Reg.getMSB() && "target is current MSB group");
- MCRegister NewPhysReg = tryAssignInGroup(VirtReg, Target);
- if (NewPhysReg) {
- LRM.assign(LI, NewPhysReg);
- Reg.notifyPhysAssignmentChanged(Target);
- return true;
- }
- }
- // We failed to find a register, re-assign the original one.
- LRM.assign(LI, OriginalPhys);
- return false;
+ unsigned Idx = TRI.getHWRegIndex(PhysReg);
+ MSBGroup PhysRegGroup = Idx >> GroupSizeLog2;
+ return Idx + NumLanes > (PhysRegGroup + 1) * GroupSize;
}
bool AMDGPUOptimizeVGPREncoding::run(MachineFunction &MF) {
@@ -1583,7 +1572,7 @@ bool AMDGPUOptimizeVGPREncoding::run(MachineFunction &MF) {
ScoreChanged.set(Candidate->Reg.getVirtIndex());
// Attempts re-assignment to a better MSB group.
- if (VRRM.tryAssignToTargetGroups(Candidate->Reg, Candidate->Targets)) {
+ if (VRRM.tryReAssign(Candidate->Reg, Candidate->Targets)) {
LLVM_DEBUG(dbgs() << " | SUCCESS: Assigned to free physical register "
<< printReg(VRM.getPhys(Candidate->Reg.getVirt()),
ST.getRegisterInfo(), 0, &MF.getRegInfo())
@@ -1603,7 +1592,7 @@ bool AMDGPUOptimizeVGPREncoding::run(MachineFunction &MF) {
return Changed;
}
-#ifndef NDEBUG
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
Printable MBBModeUsage::print(const VirtRegMap &VRM) const {
// 16 characters is enough for a virtual register with 9 digits, or to
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.h b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.h
index c1e59438929fdf..a69bc8484eb22f 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.h
@@ -13,7 +13,7 @@
namespace llvm {
class AMDGPUOptimizeVGPREncodingPass
- : public detail::PassInfoMixin<AMDGPUOptimizeVGPREncodingPass> {
+ : public OptionalPassInfoMixin<AMDGPUOptimizeVGPREncodingPass> {
public:
PreservedAnalyses run(MachineFunction &MF,
MachineFunctionAnalysisManager &MFAM);
diff --git a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
index 267efaad60c79a..861b904e054c1a 100644
--- a/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
+++ b/llvm/test/CodeGen/AMDGPU/llc-pipeline-npm.ll
@@ -249,6 +249,7 @@
; GCN-O2-NEXT: greedy<vgpr>
; GCN-O2-NEXT: amdgpu-nsa-reassign
; GCN-O2-NEXT: amdgpu-rewrite-agpr-copy-mfma
+; GCN-O2-NEXT: amdgpu-optimize-vgpr-encoding
; GCN-O2-NEXT: virt-reg-rewriter
; GCN-O2-NEXT: amdgpu-mark-last-scratch-load
; GCN-O2-NEXT: stack-slot-coloring
@@ -438,6 +439,7 @@
; GCN-O3-NEXT: greedy<vgpr>
; GCN-O3-NEXT: amdgpu-nsa-reassign
; GCN-O3-NEXT: amdgpu-rewrite-agpr-copy-mfma
+; GCN-O3-NEXT: amdgpu-optimize-vgpr-encoding
; GCN-O3-NEXT: virt-reg-rewriter
; GCN-O3-NEXT: amdgpu-mark-last-scratch-load
; GCN-O3-NEXT: stack-slot-coloring
diff --git a/llvm/unittests/Target/AMDGPU/AMDGPUOptimizeVGPREncodingTest.cpp b/llvm/unittests/Target/AMDGPU/AMDGPUOptimizeVGPREncodingTest.cpp
index 185e019b5bb5e9..d3c0c562826963 100644
--- a/llvm/unittests/Target/AMDGPU/AMDGPUOptimizeVGPREncodingTest.cpp
+++ b/llvm/unittests/Target/AMDGPU/AMDGPUOptimizeVGPREncodingTest.cpp
@@ -20,6 +20,7 @@
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/Passes/PassBuilder.h"
#include "llvm/Support/MathExtras.h"
+#include "llvm/Support/raw_ostream.h"
#include "gtest/gtest.h"
using namespace llvm;
@@ -36,16 +37,26 @@ using OprdType = unsigned;
static constexpr unsigned NumOprdTypes = 4;
/// A VGPR definition with an initial physical mapping.
-struct VGPRDef {
+struct VirtVGPRDef {
+ /// The register's name, with the leading '%'.
+ StringRef RegName;
/// The register's class, in MIR-spelling (e.g., "vgpr_32").
StringRef RegClass;
+ /// MSB group to which the physical register must belong.
+ MSBGroup Group;
/// Index of the physical register, offset within the MSB group to the first
/// non-reserved VGPR.
unsigned PhysRegIdx;
- /// MSB group to which the physical register must belong.
- MSBGroup Group;
+
+ VirtVGPRDef(StringRef RegDef, MSBGroup Group, unsigned PhysRegIdx)
+ : Group(Group), PhysRegIdx(PhysRegIdx) {
+ std::tie(RegName, RegClass) = RegDef.split(':');
+ }
};
+/// A physical VGPR definition.
+using PhysVGPRDef = StringRef;
+
/// An "abstract" MODE-using instruction. We don't care about their exact nature
/// for the sake of unit tests, just that they have specific operand sets that
/// get their MSBs from MODE.
@@ -55,12 +66,32 @@ struct VGPRDef {
/// registers (virtual or physical) in MIR-spelling (e.g.,
/// "%0"/"$vgpr2_vgpr3"). Virtual registers used in source positions must have
/// been defined through a \ref VGPRDef or a previous \p ModeUsingInstr in the
-/// instruction order. Physical registers may be reserved ones.
+/// instruction order. Virtual registers are assumed to be of class "vgpr_32"
+/// unless their class is explicitly provided (e.g., "%0:vgpr_32_lo256").
+/// Physical registers may be reserved ones.
struct ModeUsingInstr {
- /// Creates a V_FMA_F32_e64 instruction.
- static ModeUsingInstr vFMA(StringRef Dst, StringRef Src0, StringRef Src1,
- StringRef Src2) {
- ModeUsingInstr Instr(InstrType::V_FMA);
+ /// Creates a VOP1 instruction.
+ static ModeUsingInstr createVOP1(StringRef Dst, StringRef Src0) {
+ ModeUsingInstr Instr(InstrType::VOP1);
+ Instr.Regs[SRC0] = Src0;
+ Instr.Regs[DST] = Dst;
+ return Instr;
+ }
+
+ /// Creates a VOP2 instruction.
+ static ModeUsingInstr createVOP2(StringRef Dst, StringRef Src0,
+ StringRef Src1) {
+ ModeUsingInstr Instr(InstrType::VOP2);
+ Instr.Regs[SRC0] = Src0;
+ Instr.Regs[SRC1] = Src1;
+ Instr.Regs[DST] = Dst;
+ return Instr;
+ }
+
+ /// Creates a VOP3 instruction.
+ static ModeUsingInstr createVOP3(StringRef Dst, StringRef Src0,
+ StringRef Src1, StringRef Src2) {
+ ModeUsingInstr Instr(InstrType::VOP3);
Instr.Regs[SRC0] = Src0;
Instr.Regs[SRC1] = Src1;
Instr.Regs[SRC2] = Src2;
@@ -68,28 +99,128 @@ struct ModeUsingInstr {
return Instr;
}
+ /// Creates a VOP2 instruction whose src2 is tied to its destination.
+ static ModeUsingInstr createVOP2Tied(StringRef DstSrc2, StringRef Src0,
+ StringRef Src1) {
+ ModeUsingInstr Instr(InstrType::VOP2Tied);
+ Instr.Regs[SRC0] = Src0;
+ Instr.Regs[SRC1] = Src1;
+ Instr.Regs[SRC2] = DstSrc2;
+ Instr.Regs[DST] = DstSrc2;
+ return Instr;
+ }
+
+ /// Creates a VOPC instruction.
+ static ModeUsingInstr createVOPC(StringRef Src0, StringRef Src1) {
+ ModeUsingInstr Instr(InstrType::VOPC);
+ Instr.Regs[SRC0] = Src0;
+ Instr.Regs[SRC1] = Src1;
+ return Instr;
+ }
+
+ /// Creates a VOPD instruction whose X and Y components both have a
+ /// destination, src0 and src1.
+ static ModeUsingInstr createVOPD(StringRef DstX, StringRef Src0X,
+ StringRef Src1X, StringRef DstY,
+ StringRef Src0Y, StringRef Src1Y) {
+ ModeUsingInstr Instr(InstrType::VOPD);
+ Instr.Regs[SRC0] = Src0X;
+ Instr.Regs[SRC1] = Src1X;
+ Instr.Regs[DST] = DstX;
+ Instr.RegsY[SRC0] = Src0Y;
+ Instr.RegsY[SRC1] = Src1Y;
+ Instr.RegsY[DST] = DstY;
+ return Instr;
+ }
+
+ /// Creates a VOPD instruction whose X component has no src1 but whose Y
+ /// component does.
+ static ModeUsingInstr createVOPDNoSrc1X(StringRef DstX, StringRef Src0X,
+ StringRef DstY, StringRef Src0Y,
+ StringRef Src1Y) {
+ ModeUsingInstr Instr(InstrType::VOPDNoSrc1X);
+ Instr.Regs[SRC0] = Src0X;
+ Instr.Regs[DST] = DstX;
+ Instr.RegsY[SRC0] = Src0Y;
+ Instr.RegsY[SRC1] = Src1Y;
+ Instr.RegsY[DST] = DstY;
+ return Instr;
+ }
+
/// Serializes the instruction to string.
std::string toString() const {
- SmallString<256> S;
+ std::string Instr;
+ raw_string_ostream OS(Instr);
switch (Ty) {
- case InstrType::V_FMA:
- return (Twine(Regs[DST]) + ":vgpr_32 = V_FMA_F32_e64 0, " +
- Twine(Regs[SRC0]) + ":vgpr_32, 0, " + Twine(Regs[SRC1]) +
- ":vgpr_32, 0, " + Twine(Regs[SRC2]) +
- ":vgpr_32, 0, 0, implicit $mode, implicit $exec")
- .toNullTerminatedStringRef(S)
- .str();
+ case InstrType::VOP1:
+ OS << regToString(DST) << " = nofpexcept V_CEIL_F32_e32 "
+ << regToString(SRC0) << ", implicit $mode, implicit $exec";
+ break;
+ case InstrType::VOP2:
+ OS << regToString(DST) << " = nofpexcept V_ADD_F32_e64 0, "
+ << regToString(SRC0) << ", 0, " << regToString(SRC1)
+ << ", 0, 0, implicit $mode, implicit $exec";
+ break;
+ case InstrType::VOP3:
+ OS << regToString(DST) << " = V_FMA_F32_e64 0, " << regToString(SRC0)
+ << ", 0, " << regToString(SRC1) << ", 0, " << regToString(SRC2)
+ << ", 0, 0, implicit $mode, implicit $exec";
+ break;
+ case InstrType::VOP2Tied:
+ OS << regToString(DST) << " = nofpexcept V_FMAC_F32_e32 "
+ << regToString(SRC0) << ", " << regToString(SRC1) << ", "
+ << regToString(SRC2) << ", implicit $mode, implicit $exec";
+ break;
+ case InstrType::VOPC:
+ OS << "V_CMPX_EQ_I32_e32 " << regToString(SRC0) << ", "
+ << regToString(SRC1)
+ << ", implicit-def $exec, implicit-def $vcc, implicit $exec";
+ break;
+ case InstrType::VOPD:
+ OS << regToString(DST) << ", " << regToString(DST, true)
+ << " = V_DUAL_SUB_F32_e32_X_MUL_F32_e32_gfx1250 " << regToString(SRC0)
+ << ", " << regToString(SRC1) << ", " << regToString(SRC0, true) << ", "
+ << regToString(SRC1, true) << ", implicit $mode, implicit $exec";
+ break;
+ case InstrType::VOPDNoSrc1X:
+ OS << regToString(DST) << ", " << regToString(DST, true)
+ << " = V_DUAL_MOV_B32_e32_X_ADD_F32_e32_gfx1250 " << regToString(SRC0)
+ << ", " << regToString(SRC0, true) << ", " << regToString(SRC1, true)
+ << ", implicit $mode, implicit $exec";
+ break;
}
+ return Instr;
+ }
+
+ /// Serializes the register of operand type \p Oprd, from the Y component of
+ /// VOPD instructions if \p IsY is true.
+ std::string regToString(OprdType Oprd, bool IsY = false) const {
+ StringRef Reg = IsY ? RegsY[Oprd] : Regs[Oprd];
+ return Reg.str() +
+ (Reg.starts_with("%") && !Reg.contains(':') ? ":vgpr_32" : "");
}
private:
enum { SRC0 = 0, SRC1 = 1, SRC2 = 2, DST = 3 };
- std::array<StringRef, NumOprdTypes> Regs;
-
- enum class InstrType { V_FMA };
+ /// Registers of each operand type. For VOPD instructions, \ref Regs holds
+ /// the X component's registers and \ref RegsY the Y component's.
+ std::array<StringRef, NumOprdTypes> Regs, RegsY;
+
+ enum class InstrType {
+ VOP1,
+ VOP2,
+ VOP2Tied,
+ VOP3,
+ VOPC,
+ VOPD,
+ VOPDNoSrc1X,
+ };
InstrType Ty;
- ModeUsingInstr(InstrType Ty) : Ty(Ty) { Regs.fill(""); }
+ ModeUsingInstr(InstrType Ty) : Ty(Ty) {
+ Regs.fill("");
+ RegsY.fill("");
+ }
};
class AMDGPUOptimizeVGPREncodingTest : public AMDGPUCodeGenTestBase {
@@ -103,14 +234,13 @@ class AMDGPUOptimizeVGPREncodingTest : public AMDGPUCodeGenTestBase {
const SIRegisterInfo *TRI;
const SIInstrInfo *TII;
RegisterClassInfo RegClassInfo;
- unsigned NumFreeVGPRsPerGroup;
void SetUp() override { setUpImpl("amdgpu12.50--", "", ""); }
- /// Marks every VGPR as reserved except the top NumFreeVGPRsPerGroup registers
- /// of each MSB group, so that the pass only ever has that small window in
- /// each MSB group to (re-)assign registers into.
- void reserveVGPRs() {
+ /// Marks every VGPR as reserved except the top \p NumFreeVGPRsPerGroup
+ /// registers of each MSB group, so that the pass only ever has that small
+ /// window in each MSB group to (re-)assign registers into.
+ void reserveVGPRs(unsigned NumFreeVGPRsPerGroup) {
const TargetRegisterClass &RC = AMDGPU::VGPR_32RegClass;
for (unsigned I = 0, E = RC.getNumRegs(); I != E; ++I) {
if (I % MSBGroupSize < MSBGroupSize - NumFreeVGPRsPerGroup)
@@ -128,12 +258,24 @@ class AMDGPUOptimizeVGPREncodingTest : public AMDGPUCodeGenTestBase {
/// register re-assignments and make tests more trackable, all VGPRs but the
/// top \p NumFreeVGPRsPerGroup in each MSB group are marked reserved.
bool createMIRAndAssign(
- ArrayRef<VGPRDef> RegDefs,
+ ArrayRef<VirtVGPRDef> VirtDefs, ArrayRef<PhysVGPRDef> PhysDefs,
ArrayRef<const SmallVectorImpl<ModeUsingInstr> *> MIRBlocks,
unsigned NumFreeVGPRsPerGroup) {
assert(NumFreeVGPRsPerGroup < MSBGroupSize);
- std::string MIRString = R"MIR(
+ // Physical registers need to be marked as live-ins in all blocks.
+ std::string LiveIns;
+ if (!PhysDefs.empty()) {
+ raw_string_ostream LiveInsOS(LiveIns);
+ LiveInsOS << " liveins: ";
+ for (PhysVGPRDef PhysDef : drop_end(PhysDefs))
+ LiveInsOS << PhysDef << ", ";
+ LiveInsOS << PhysDefs.back() << '\n';
+ }
+
+ std::string MIRString;
+ raw_string_ostream OS(MIRString);
+ OS << R"MIR(
--- |
define amdgpu_kernel void @func() #0 {
ret void
@@ -148,26 +290,29 @@ tracksRegLiveness: true
body: |
bb.0:
)MIR";
+ OS << LiveIns;
// All registrer definitions go in the entry block. All MODE-using
// instructions go in subsequent blocks, and finally an exit block with
// implicit uses of all registers to extend their live-range over the entire
// interesting part of the function.
std::string RegisterUses;
- for (const auto &[VirtRegIdx, Assignment] : enumerate(RegDefs)) {
+ raw_string_ostream RegUseOS(RegisterUses);
+
+ for (const VirtVGPRDef VirtDef : VirtDefs) {
// Each virtual register gets an IMPLICIT_DEF.
- MIRString += " %" + std::to_string(VirtRegIdx) + ':' +
- Assignment.RegClass.str() + " = IMPLICIT_DEF\n";
- // Accumulate uses for later.
- RegisterUses += ", implicit %" + std::to_string(VirtRegIdx);
+ OS << " " << VirtDef.RegName << ':' << VirtDef.RegClass
+ << " = IMPLICIT_DEF\n";
+ // Accumulate uses for the last block.
+ RegUseOS << ", implicit " << VirtDef.RegName;
}
for (const auto &[Idx, Block] : enumerate(MIRBlocks)) {
- MIRString += " bb." + std::to_string(Idx + 1) + ":\n";
+ OS << " bb." << Idx + 1 << ":\n" << LiveIns;
for (const ModeUsingInstr &Instr : *Block)
- MIRString += " " + Instr.toString() + '\n';
+ OS << " " << Instr.toString() << '\n';
}
- MIRString += " bb." + std::to_string(MIRBlocks.size() + 1) +
- ":\n S_NOP 0" + RegisterUses + "\n...\n";
+ OS << " bb." << MIRBlocks.size() + 1 << ":\n S_NOP 0" << RegisterUses
+ << "\n S_ENDPGM 0\n...\n";
if (!parseMIR(MIRString))
return false;
@@ -184,17 +329,24 @@ body: |
// Reserve VGPRs except the last NumFreeVGPRsPerGroup in each group, then
// create an initial virtual-to-physical assignment using free physical
// registers.
- this->NumFreeVGPRsPerGroup = NumFreeVGPRsPerGroup;
- reserveVGPRs();
- for (const auto &[VirtRegIdx, Assignment] : enumerate(RegDefs))
- assign(VirtRegIdx, Assignment.PhysRegIdx, Assignment.Group);
+ reserveVGPRs(NumFreeVGPRsPerGroup);
+ for (const auto &[VirtRegIdx, Assignment] : enumerate(VirtDefs)) {
+ OriginalAssignments.insert(
+ {Assignment.RegName,
+ {Register::index2VirtReg(VirtRegIdx), Assignment.Group}});
+ assign(VirtRegIdx, Assignment.PhysRegIdx, Assignment.Group,
+ NumFreeVGPRsPerGroup);
+ }
+
+ OriginalNumModeSet = getNumModeSets();
return true;
}
/// Assigns virtual register with index \p VirtRegIdx to physical
- /// register \p PhysRegIdx (offset within the MSB group to the first
- /// non-reserved VGPR) in \p Group.
- void assign(unsigned VirtRegIdx, unsigned PhysRegIdx, MSBGroup Group) {
+ /// register \p PhysRegIdx (offset within the MSB group to the last \p
+ /// NumFreeVGPRsPerGroup registers) in \p Group.
+ void assign(unsigned VirtRegIdx, unsigned PhysRegIdx, MSBGroup Group,
+ unsigned NumFreeVGPRsPerGroup) {
assert(Group < NumMSBGroups && "invalid MSB group");
Register VirtReg = Register::index2VirtReg(VirtRegIdx);
@@ -221,20 +373,20 @@ body: |
return TRI->getHWRegIndex(PhysReg) >> 8;
}
- /// Counts the number of S_SET_VGPR_MSB instructions the \ref MF will require.
- unsigned countSetModeInstrs() const {
+ /// Counts the number of MODE-setting instructions the \ref MF will require.
+ unsigned getNumModeSets() const {
unsigned NumSetModeInstrs = 0;
for (const MachineBasicBlock &MBB : *MF)
NumSetModeInstrs += countSetModeInstrs(MBB);
return NumSetModeInstrs;
}
- /// Counts the number of S_SET_VGPR_MSB instructions \p MBB will require.
+ /// Counts the number of MODE-setting instructions \p MBB will require.
unsigned countSetModeInstrs(const MachineBasicBlock &MBB) const {
unsigned NumSetModeInstrs = 0;
// A std::nullopt for a particular operand type means that there exists a
- // previous S_SET_VGPR_MSB instruction whose group for that operand type is
+ // previous MODE-setting instruction whose group for that operand type is
// not yet constrained i.e., onto which we can piggyback a later group
// requirement.
std::array<std::optional<MSBGroup>, NumOprdTypes> CurrentGroups, MIGroups;
@@ -291,7 +443,7 @@ body: |
}
// Merge group requirements from the MI with the current ones, and
- // determine whether we need a new S_SET_VGPR_MSB.
+ // determine whether we need a new MODE-setting.
for (auto [Current, Requirement] : zip(CurrentGroups, MIGroups)) {
if (Current.has_value()) {
if (Requirement.has_value() && *Current != *Requirement) {
@@ -300,14 +452,14 @@ body: |
break;
}
} else {
- // Piggyback into previous S_SET_VGPR_MSB instruction.
+ // Piggyback into previous MODE-setting instruction.
Current = Requirement;
}
}
}
// Groups must end with all MSBs set to the default group so an extra
- // S_SET_VGPR_MSB instruction would be inserted if that is not the case.
+ // MODE-setting instruction would be inserted if that is not the case.
for (const std::optional<MSBGroup> &Group : CurrentGroups) {
if (Group.has_value() && *Group != DefaultGroup) {
++NumSetModeInstrs;
@@ -318,90 +470,246 @@ body: |
return NumSetModeInstrs;
}
- /// Runs the optimization pass, expecting \p ExpectModeSetBefore
- /// S_SET_VGPR_MSB instructions to be required in the function before it, and
- /// \p ExpectModeSetAfter after it.
- void runWithExpectation(unsigned ExpectModeSetBefore,
- unsigned ExpectModeSetAfter) {
- unsigned ActualModeSetBefore = countSetModeInstrs();
- EXPECT_EQ(ExpectModeSetBefore, ActualModeSetBefore);
+ void runPass() { AMDGPUOptimizeVGPREncodingPass().run(*MF, MFAM); }
+
+ /// Expects \p ExpectModeSetBefore MODE-setting instructions to be required in
+ /// the function before the pass, and \p ExpectModeSetAfter after it.
+ void expectNumModeSetChange(unsigned ExpectModeSetBefore,
+ unsigned ExpectModeSetAfter) {
+ EXPECT_EQ(ExpectModeSetBefore, OriginalNumModeSet) << "before the pass";
+ EXPECT_EQ(ExpectModeSetAfter, getNumModeSets()) << "after the pass";
+ }
- AMDGPUOptimizeVGPREncodingPass Pass;
- Pass.run(*MF, MFAM);
+ /// Expects registers in \p Changes (identified by name, first element) to
+ /// have moved to the group indicated by the second element. If \p Exhaustive
+ /// is true, also expects that all other registers have not moved to a
+ /// different group than the one they were assigned to at the beginning.
+ void expectAssignmentChanges(ArrayRef<std::pair<StringRef, MSBGroup>> Changes,
+ bool Exhaustive) const {
+ SmallDenseSet<StringRef, 4> ChangedRegs;
+ for (const auto &[RegName, ExpectedGroup] : Changes) {
+ auto Original = OriginalAssignments.find(RegName);
+ ASSERT_NE(Original, OriginalAssignments.end())
+ << "virtreg " << RegName << " does not exist";
+ EXPECT_EQ(ExpectedGroup, getVGPRGroup(Original->second.VirtReg))
+ << "for virtreg " << RegName;
+
+ if (Exhaustive)
+ ChangedRegs.insert(RegName);
+ }
- unsigned ActualModeSetAfter = countSetModeInstrs();
- EXPECT_EQ(ExpectModeSetAfter, ActualModeSetAfter);
+ if (!Exhaustive)
+ return;
+ for (const auto &[RegName, Assignment] : OriginalAssignments) {
+ if (ChangedRegs.contains(RegName))
+ continue;
+ EXPECT_EQ(Assignment.Group, getVGPRGroup(Assignment.VirtReg))
+ << "assignment of virtreg " << RegName << " unexpectedly changed";
+ }
}
+
+private:
+ struct VirtRegAndGroup {
+ Register VirtReg;
+ MSBGroup Group;
+ };
+ DenseMap<StringRef, VirtRegAndGroup> OriginalAssignments;
+ unsigned OriginalNumModeSet;
};
-/// All registers of the VFMA are in group 1, requiring S_SET_VGPR_MSB around
-/// the instruction because all operand types start (and must end) in the
-/// default group.
+/// All registers of the VOP3 are in group 1, requiring MODE-setting
+/// instructions around it because all operand types start (and must end) in the
+/// default group. All register should be re-assigned to the default group.
TEST_F(AMDGPUOptimizeVGPREncodingTest, ReassignToDefaultGroup) {
- SmallVector<VGPRDef> Registers{
- {"vgpr_32", 0, 1}, // %0
- {"vgpr_32", 1, 1}, // %1
- {"vgpr_32", 2, 1}, // %2
- {"vgpr_32", 3, 1}, // %3
+ SmallVector<VirtVGPRDef> VirtDefs{
+ {"%0:vgpr_32", 1, 0},
+ {"%1:vgpr_32", 1, 1},
+ {"%2:vgpr_32", 1, 2},
+ {"%3:vgpr_32", 1, 3},
};
SmallVector<ModeUsingInstr> Instructions{
- ModeUsingInstr::vFMA("%0", "%1", "%2", "%3")};
+ ModeUsingInstr::createVOP3("%0", "%1", "%2", "%3")};
- ASSERT_TRUE(createMIRAndAssign(Registers, {&Instructions}, 8));
- runWithExpectation(2, 0);
+ ASSERT_TRUE(createMIRAndAssign(VirtDefs, {}, {&Instructions}, 4));
+ runPass();
+ expectNumModeSetChange(2, 0);
}
-/// Each VFMA has all its registers in the same non-default group, and the two
-/// VFMAs have different groups. All registers should all be re-assigned to the
+/// Each VOP3 has all its registers in the same non-default group, and the two
+/// VOP3s have different groups. All registers should be re-assigned to the
/// default group which has just enough free registers. The pass must avoid a
-/// local-maxima where all registers end up in one of the VFMA's group.
-TEST_F(AMDGPUOptimizeVGPREncodingTest, ReassignToDefaultGroupVFMAConflict) {
- SmallVector<VGPRDef> Registers{
- {"vgpr_32", 0, 1}, // %0
- {"vgpr_32", 1, 1}, // %1
- {"vgpr_32", 2, 1}, // %2
- {"vgpr_32", 3, 1}, // %3
- {"vgpr_32", 0, 2}, // %4
- {"vgpr_32", 1, 2}, // %5
- {"vgpr_32", 2, 2}, // %6
- {"vgpr_32", 3, 2}, // %7
+/// local-maximum where all registers end up in one of the VOP3's group.
+TEST_F(AMDGPUOptimizeVGPREncodingTest, ReassignToDefaultGroupVOP3Conflict) {
+ SmallVector<VirtVGPRDef> VirtDefs{
+ {"%group1_0:vgpr_32", 1, 0}, {"%group1_1:vgpr_32", 1, 1},
+ {"%group1_2:vgpr_32", 1, 2}, {"%group1_3:vgpr_32", 1, 3},
+ {"%group2_0:vgpr_32", 2, 0}, {"%group2_1:vgpr_32", 2, 1},
+ {"%group2_2:vgpr_32", 2, 2}, {"%group2_3:vgpr_32", 2, 3},
};
SmallVector<ModeUsingInstr> Instructions{
- ModeUsingInstr::vFMA("%0", "%1", "%2", "%3"),
- ModeUsingInstr::vFMA("%4", "%5", "%6", "%7")};
- ASSERT_TRUE(createMIRAndAssign(Registers, {&Instructions}, 8));
- runWithExpectation(3, 0);
+ ModeUsingInstr::createVOP3("%group1_0", "%group1_1", "%group1_2",
+ "%group1_3"),
+ ModeUsingInstr::createVOP3("%group2_0", "%group2_1", "%group2_2",
+ "%group2_3")};
+ ASSERT_TRUE(createMIRAndAssign(VirtDefs, {}, {&Instructions}, 8));
+ runPass();
+ expectNumModeSetChange(3, 0);
}
-/// Each VFMA has each of its register in a different group, and consecutive
-/// VFMAs have matching operands in different groups. There is a single free
+/// Each VOP3 has each of its register in a different group, and consecutive
+/// VOP3 have matching operands in different groups. There is a single free
/// register in each group, which makes it hard for the pass to find
/// re-assignments, even though there is a solution that only requires a single
-/// S_SET_VGPR_MSB.
+/// MODE-setting instruction.
TEST_F(AMDGPUOptimizeVGPREncodingTest, SingleFreeRegPerGroup) {
- SmallVector<VGPRDef> Registers{
- {"vgpr_32", 0, 0}, // %0
- {"vgpr_32", 0, 1}, // %1
- {"vgpr_32", 0, 2}, // %2
- {"vgpr_32", 0, 3}, // %3
- {"vgpr_32", 1, 1}, // %4
- {"vgpr_32", 1, 2}, // %5
- {"vgpr_32", 1, 3}, // %6
- {"vgpr_32", 1, 0}, // %7
- {"vgpr_32", 2, 2}, // %8
- {"vgpr_32", 2, 3}, // %9
- {"vgpr_32", 2, 0}, // %10
- {"vgpr_32", 2, 1}, // %11
- {"vgpr_32", 3, 3}, // %12
- {"vgpr_32", 3, 0}, // %13
- {"vgpr_32", 3, 1}, // %14
- {"vgpr_32", 3, 2}, // %15
+ SmallVector<VirtVGPRDef> VirtDefs{
+ {"%0:vgpr_32", 0, 0}, {"%1:vgpr_32", 1, 0}, {"%2:vgpr_32", 2, 0},
+ {"%3:vgpr_32", 3, 0}, {"%4:vgpr_32", 1, 1}, {"%5:vgpr_32", 2, 1},
+ {"%6:vgpr_32", 3, 1}, {"%7:vgpr_32", 0, 1}, {"%8:vgpr_32", 2, 2},
+ {"%9:vgpr_32", 3, 2}, {"%10:vgpr_32", 0, 2}, {"%11:vgpr_32", 1, 2},
+ {"%12:vgpr_32", 3, 3}, {"%13:vgpr_32", 0, 3}, {"%14:vgpr_32", 1, 3},
+ {"%15:vgpr_32", 2, 3},
+ };
+ SmallVector<ModeUsingInstr> Instructions{
+ ModeUsingInstr::createVOP3("%0", "%1", "%2", "%3"),
+ ModeUsingInstr::createVOP3("%4", "%5", "%6", "%7"),
+ ModeUsingInstr::createVOP3("%8", "%9", "%10", "%11"),
+ ModeUsingInstr::createVOP3("%12", "%13", "%14", "%15")};
+ ASSERT_TRUE(createMIRAndAssign(VirtDefs, {}, {&Instructions}, 5));
+ runPass();
+ expectNumModeSetChange(5, 3);
+}
+
+/// Tests re-assignment priority w.r.t. the number of neighbors in each
+/// group. There is a single free register in the default group. %group2 should
+/// have re-assignment priority because it has twice as many neighboring
+/// relationships with it.
+TEST_F(AMDGPUOptimizeVGPREncodingTest, HigherNumberOfOccurencesWins) {
+ SmallVector<VirtVGPRDef> VirtDefs{
+ {"%group1:vgpr_32", 1, 0},
+ {"%group2:vgpr_32", 2, 0},
+ };
+ SmallVector<ModeUsingInstr> Instructions{
+ ModeUsingInstr::createVOP2("%group1", "%group2", "%group2"),
+ };
+ ASSERT_TRUE(createMIRAndAssign(VirtDefs, {}, {&Instructions}, 1));
+ runPass();
+ expectAssignmentChanges({{"%group2", 0}}, /*Exhaustive=*/true);
+}
+
+/// Tests re-assignment priority in the presence of neighbors in multiple
+/// groups. There is a single free register in the default group. %clearMoveTo0
+/// should have re-assignment priority because it has no neighboring
+/// relationships with other groups, whereas %group1 and %group2 have.
+TEST_F(AMDGPUOptimizeVGPREncodingTest, ClearerTargetWins) {
+ SmallVector<VirtVGPRDef> VirtDefs{
+ {"%group1:vgpr_32", 1, 0},
+ {"%group2:vgpr_32", 2, 0},
+ {"%clearMoveTo0:vgpr_32", 3, 0},
+ };
+ SmallVector<ModeUsingInstr> Instructions{
+ ModeUsingInstr::createVOP1("%group1", "%clearMoveTo0"),
+ ModeUsingInstr::createVOP1("%group2", "%clearMoveTo0"),
+ };
+ ASSERT_TRUE(createMIRAndAssign(VirtDefs, {}, {&Instructions}, 1));
+ runPass();
+ expectAssignmentChanges({{"%clearMoveTo0", 0}}, /*Exhaustive=*/true);
+}
+
+/// Tests re-assignment priority in the presence of pinned neighbors. There is a
+/// single free register in group 0 (%vgpr255). All virtual registers not in
+/// group 0 would benefit from a move to it. %goodPin's neighbors are all pinned
+/// and in group 0 so it gets priority.
+TEST_F(AMDGPUOptimizeVGPREncodingTest, GoodPinWins) {
+ SmallVector<VirtVGPRDef> VirtDefs{
+ {"%group0:vgpr_32", 0, 0},
+ {"%goodPin:vgpr_32", 1, 0},
+ {"%badPin:vgpr_32", 1, 1},
+ {"%free:vgpr_32", 2, 0},
+ };
+ SmallVector<ModeUsingInstr> Instructions{
+ ModeUsingInstr::createVOP3("$vgpr0", "%group0", "%group0", "$vgpr256"),
+ ModeUsingInstr::createVOP3("%goodPin", "%free", "%badPin", "%badPin"),
+ ModeUsingInstr::createVOP3("$vgpr0", "%group0", "%group0", "%group0"),
+ };
+ ASSERT_TRUE(
+ createMIRAndAssign(VirtDefs, {"$vgpr0", "$vgpr256"}, {&Instructions}, 2));
+ runPass();
+ expectAssignmentChanges({{"%goodPin", 0}}, /*Exhaustive=*/true);
+}
+
+/// The tied src2 use of a VOP2 only depends on the MSB group of the
+/// destination; it must not be considered a src2 occurrence. %moveTo0 should
+/// move to the adjacent dst operands' group ($vgpr0, group 0) instead
+/// of the adjacent src2 operands' group ($vgpr256, group 1).
+TEST_F(AMDGPUOptimizeVGPREncodingTest, TiedSrc2FollowsDst) {
+ SmallVector<VirtVGPRDef> VirtDefs{
+ {"%moveTo0:vgpr_32", 2, 0},
+ };
+ SmallVector<ModeUsingInstr> Instructions{
+ ModeUsingInstr::createVOP3("$vgpr0", "$vgpr0", "$vgpr256", "$vgpr256"),
+ ModeUsingInstr::createVOP2Tied("%moveTo0", "$vgpr0", "%moveTo0"),
+ ModeUsingInstr::createVOP3("$vgpr0", "$vgpr0", "$vgpr0", "$vgpr256"),
+ };
+ ASSERT_TRUE(
+ createMIRAndAssign(VirtDefs, {"$vgpr0", "$vgpr256"}, {&Instructions}, 1));
+ runPass();
+ expectAssignmentChanges({{"%moveTo0", 0}}, /*Exhaustive=*/true);
+}
+
+/// All registers of the VOPD are in group 1, requiring MODE-setting
+/// instructions around it. Re-assigning them all to the default group would be
+/// profitable, but VOPD operands are pinned so nothing should change.
+TEST_F(AMDGPUOptimizeVGPREncodingTest, VOPDRegsArePinned) {
+ SmallVector<VirtVGPRDef> VirtDefs{
+ {"%dstX:vgpr_32", 1, 0}, {"%src0X:vgpr_32", 1, 1},
+ {"%src1X:vgpr_32", 1, 2}, {"%dstY:vgpr_32", 1, 3},
+ {"%src0Y:vgpr_32", 1, 4}, {"%src1Y:vgpr_32", 1, 5},
};
SmallVector<ModeUsingInstr> Instructions{
- ModeUsingInstr::vFMA("%0", "%1", "%2", "%3"),
- ModeUsingInstr::vFMA("%4", "%5", "%6", "%7"),
- ModeUsingInstr::vFMA("%8", "%9", "%10", "%11"),
- ModeUsingInstr::vFMA("%12", "%13", "%14", "%15")};
- ASSERT_TRUE(createMIRAndAssign(Registers, {&Instructions}, 5));
- runWithExpectation(5, 5);
+ ModeUsingInstr::createVOPD("%dstX", "%src0X", "%src1X", "%dstY", "%src0Y",
+ "%src1Y"),
+ };
+ ASSERT_TRUE(createMIRAndAssign(VirtDefs, {}, {&Instructions}, 6));
+ runPass();
+ expectAssignmentChanges({}, /*Exhaustive=*/true);
+}
+
+/// The X component of the VOPDs has no src1 but the Y component does, so the
+/// latter provides the src1 MSB group of the VOPDs. %moveTo1's src1 neighbors
+/// are thus $vgpr256 (group 1) on both sides, and it should move from group 2
+/// to group 1.
+TEST_F(AMDGPUOptimizeVGPREncodingTest, VOPDSecondComponentOprd) {
+ SmallVector<VirtVGPRDef> VirtDefs{
+ {"%moveTo1:vgpr_32", 2, 0},
+ };
+ SmallVector<ModeUsingInstr> Instructions{
+ ModeUsingInstr::createVOPDNoSrc1X("$vgpr0", "$vgpr0", "$vgpr0", "$vgpr0",
+ "$vgpr256"),
+ ModeUsingInstr::createVOPC("$vgpr0", "%moveTo1"),
+ ModeUsingInstr::createVOPDNoSrc1X("$vgpr0", "$vgpr0", "$vgpr0", "$vgpr0",
+ "$vgpr256"),
+ };
+ ASSERT_TRUE(
+ createMIRAndAssign(VirtDefs, {"$vgpr0", "$vgpr256"}, {&Instructions}, 1));
+ runPass();
+ expectAssignmentChanges({{"%moveTo1", 1}}, /*Exhaustive=*/true);
+}
+
+/// Registers whose class is confined to the first 256 VGPRs cannot leave the
+/// first MSB group so they are pinned. %lo256's and %moveTo1's neighbors are
+/// all in group 1, but only %moveTo1 should move there.
+TEST_F(AMDGPUOptimizeVGPREncodingTest, Lo256RegsArePinned) {
+ SmallVector<VirtVGPRDef> VirtDefs{
+ {"%lo256:vgpr_32_lo256", 0, 0},
+ {"%moveTo1:vgpr_32", 0, 1},
+ };
+ SmallVector<ModeUsingInstr> Instructions{
+ ModeUsingInstr::createVOP1("$vgpr256", "$vgpr256"),
+ ModeUsingInstr::createVOP1("%lo256:vgpr_32_lo256", "%moveTo1"),
+ ModeUsingInstr::createVOP1("$vgpr256", "$vgpr256"),
+ };
+ ASSERT_TRUE(createMIRAndAssign(VirtDefs, {"$vgpr256"}, {&Instructions}, 2));
+ runPass();
+ expectAssignmentChanges({{"%moveTo1", 1}}, /*Exhaustive=*/true);
}
>From 83ef1d5004796be70319a90287ee0993d5f84f94 Mon Sep 17 00:00:00 2001
From: Lucas Ramirez <lucas.rami at proton.me>
Date: Fri, 2 Oct 2026 14:52:53 +0000
Subject: [PATCH 5/5] Fix heap issue / bump->set
---
.../AMDGPU/AMDGPUOptimizeVGPREncoding.cpp | 57 +++++++++++--------
1 file changed, 32 insertions(+), 25 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
index 5111c1a0dc0f79..0d133831426c43 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUOptimizeVGPREncoding.cpp
@@ -561,10 +561,12 @@ class OptRegCandidate {
/// Returns the current epoch.
unsigned getEpoch() const { return Epoch; }
- /// Bumps the epoch to \p NewEpoch, which must be higher than the current one.
- void bumpEpoch(unsigned NewEpoch) {
- assert(Epoch < NewEpoch && "epoch must increase");
+ /// Sets the epoch to \p NewEpoch, returning whether the epoch changed.
+ bool setEpoch(unsigned NewEpoch) {
+ assert(Epoch <= NewEpoch && "epoch must not decrease");
+ const bool SameEpoch = Epoch == NewEpoch;
Epoch = NewEpoch;
+ return SameEpoch;
}
bool operator<(const OptRegCandidate &Other) const;
@@ -1394,42 +1396,48 @@ void MaxHeap::reorderIfExists(const OptReg &Reg) {
}
bool MaxHeap::siftUp(unsigned HeapIdx) {
- unsigned S = HeapToSlot[HeapIdx];
+ const unsigned SlotIdx = HeapToSlot[HeapIdx];
bool Moved = false;
while (HeapIdx != 0) {
- unsigned Parent = (HeapIdx - 1) / 2;
- if (!(Slots[HeapToSlot[Parent]].Cand < Slots[S].Cand))
+ const unsigned Parent = (HeapIdx - 1) / 2;
+ if (!(Slots[HeapToSlot[Parent]].Cand < Slots[SlotIdx].Cand))
break;
place(HeapIdx, HeapToSlot[Parent]);
HeapIdx = Parent;
Moved = true;
}
if (Moved)
- place(HeapIdx, S);
+ place(HeapIdx, SlotIdx);
return Moved;
}
bool MaxHeap::siftDown(unsigned HeapIdx) {
- unsigned S = HeapToSlot[HeapIdx];
- unsigned N = HeapToSlot.size();
+ const unsigned HeapSize = HeapToSlot.size();
+ const unsigned SlotIdx = HeapToSlot[HeapIdx];
bool Moved = false;
while (true) {
- unsigned Left = 2 * HeapIdx + 1;
- unsigned Right = Left + 1;
- unsigned Largest = HeapIdx;
- if (Left < N && Slots[S].Cand < Slots[HeapToSlot[Left]].Cand)
- Largest = Left;
- if (Right < N &&
- Slots[HeapToSlot[Largest]].Cand < Slots[HeapToSlot[Right]].Cand)
- Largest = Right;
- if (Largest == HeapIdx)
+ const unsigned Left = 2 * HeapIdx + 1;
+ const unsigned Right = Left + 1;
+
+ unsigned LargestHeapIdx = HeapIdx;
+ unsigned LargestSlotIdx = SlotIdx;
+ if (Left < HeapSize &&
+ Slots[LargestSlotIdx].Cand < Slots[HeapToSlot[Left]].Cand) {
+ LargestHeapIdx = Left;
+ LargestSlotIdx = HeapToSlot[Left];
+ }
+ if (Right < HeapSize &&
+ Slots[LargestSlotIdx].Cand < Slots[HeapToSlot[Right]].Cand)
+ LargestHeapIdx = Right;
+
+ if (LargestHeapIdx == HeapIdx)
break;
- place(HeapIdx, HeapToSlot[Largest]);
- HeapIdx = Largest;
+ place(HeapIdx, HeapToSlot[LargestHeapIdx]);
+ HeapIdx = LargestHeapIdx;
Moved = true;
}
if (Moved)
- place(HeapIdx, S);
+ place(HeapIdx, SlotIdx);
return Moved;
}
@@ -1559,13 +1567,12 @@ bool AMDGPUOptimizeVGPREncoding::run(MachineFunction &MF) {
dbgs() << " " << NeighborReg->print(VRM) << '\n';
});
- // The epoch check catches a second evaluation of the same candidate under
- // the same exact conditions, which would fail again.
- if (Candidate->getEpoch() == Epoch) {
+ // This catches a second evaluation of the same candidate under the same
+ // exact conditions, which would fail again.
+ if (Candidate->setEpoch(Epoch)) {
LLVM_DEBUG(dbgs() << " | No more useful candidates!\n");
break;
}
- Candidate->bumpEpoch(Epoch);
ScoreChanged.reset();
// The candidate's epoch changed.
More information about the llvm-commits
mailing list