[llvm] dad3e30 - [CodeGen] Fix incorrect rematerialization rollback order (#197576)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 01:33:32 PDT 2026
Author: Lucas Ramirez
Date: 2026-07-14T10:33:28+02:00
New Revision: dad3e30521b596cade7e3c516909657c44ed2840
URL: https://github.com/llvm/llvm-project/commit/dad3e30521b596cade7e3c516909657c44ed2840
DIFF: https://github.com/llvm/llvm-project/commit/dad3e30521b596cade7e3c516909657c44ed2840.diff
LOG: [CodeGen] Fix incorrect rematerialization rollback order (#197576)
This fixes an issue in the rematerializer's rollbacker wherein adjacent
MIs that were deleted through rematerializations would
sometimes---depending on the exact order in which they were
deleted---not be re-created in their original pre-rematerialization
order. While this does not impact correctness (i.e., use-def relations
are always honored), this goes against the rollbacker's intent to
re-create the MIR exactly as it was pre-rematerializations (up to slot
index changes).
Added:
Modified:
llvm/include/llvm/CodeGen/Rematerializer.h
llvm/lib/CodeGen/Rematerializer.cpp
llvm/unittests/CodeGen/RematerializerTest.cpp
Removed:
################################################################################
diff --git a/llvm/include/llvm/CodeGen/Rematerializer.h b/llvm/include/llvm/CodeGen/Rematerializer.h
index 10fe04232429a..706a4858cdb26 100644
--- a/llvm/include/llvm/CodeGen/Rematerializer.h
+++ b/llvm/include/llvm/CodeGen/Rematerializer.h
@@ -14,15 +14,12 @@
#ifndef LLVM_CODEGEN_REMATERIALIZER_H
#define LLVM_CODEGEN_REMATERIALIZER_H
-#include "llvm/ADT/MapVector.h"
-#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/PointerUnion.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
-#include "llvm/CodeGen/TargetOpcodes.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
-#include <iterator>
namespace llvm {
@@ -185,10 +182,11 @@ class Rematerializer {
virtual void rematerializerNoteRegCreated(const Rematerializer &Remater,
RegisterIdx NewRegIdx) {}
- /// Called juste before register \p RegIdx is deleted from the MIR. At this
+ /// Called just before register \p RegIdx is deleted from the MIR. At this
/// point the register still exists in the MIR but no longer has any user.
- virtual void rematerializerNoteRegDeleted(const Rematerializer &Remater,
- RegisterIdx RegIdx) {}
+ virtual void
+ rematerializerNoteRegWillBeDeleted(const Rematerializer &Remater,
+ RegisterIdx RegIdx) {}
virtual ~Listener() = default;
@@ -371,15 +369,13 @@ class Rematerializer {
MachineBasicBlock::iterator InsertPos,
SmallVectorImpl<Reg::Dependency> &&Dependencies);
- /// Re-creates a previously deleted register \p RegIdx before \p InsertPos in
- /// \p DefRegion. \p DefReg must be the original virtual register that \p
- /// RegIdx used to define. Sets the new register's rematerializable
- /// dependencies to \p Dependencies (these are assumed to already exist in the
- /// MIR).
- LLVM_ABI void recreateReg(RegisterIdx RegIdx, unsigned DefRegion,
+ /// Re-creates a previously deleted register \p RegIdx before \p InsertPos,
+ /// which must be in the register's original defining region. \p DefReg must
+ /// be the original virtual register that \p RegIdx used to define.
+ /// Dependencies are assumed to already exist in the MIR.
+ LLVM_ABI void recreateReg(RegisterIdx RegIdx,
MachineBasicBlock::iterator InsertPos,
- Register DefReg,
- SmallVectorImpl<Reg::Dependency> &&Dependencies);
+ Register DefReg);
/// Transfers all users of register \p FromRegIdx in region \p UseRegion to \p
/// ToRegIdx, the latter of which must be a rematerialization of the former or
@@ -439,9 +435,9 @@ class Rematerializer {
Listen->rematerializerNoteRegCreated(*this, RegIdx);
}
- void noteRegDeleted(RegisterIdx RegIdx) const {
+ void noteRegWillBeDeleted(RegisterIdx RegIdx) const {
for (Listener *Listen : Listeners)
- Listen->rematerializerNoteRegDeleted(*this, RegIdx);
+ Listen->rematerializerNoteRegWillBeDeleted(*this, RegIdx);
}
/// Rematerializable registers identified since the rematerializer's creation,
@@ -501,7 +497,9 @@ class Rematerializer {
/// Deletes register \p RootIdx if it no longer has any user. If the register
/// is deleted, recursively deletes any of its transitive rematerializable
- /// dependencies that no longer have users as a result.
+ /// dependencies that no longer have users as a result. In case of recursive
+ /// deletion, all of a register's users are always deleted before the register
+ /// itself.
void deleteRegIfUnused(RegisterIdx RootIdx);
/// Deletes rematerializable register \p RegIdx from the DAG and relevant
@@ -523,39 +521,64 @@ class LLVM_ABI Rollbacker : public Rematerializer::Listener {
void rematerializerNoteRegCreated(const Rematerializer &Remater,
RegisterIdx RegIdx) override;
- void rematerializerNoteRegDeleted(const Rematerializer &Remater,
- RegisterIdx RegIdx) override;
+ void rematerializerNoteRegWillBeDeleted(const Rematerializer &Remater,
+ RegisterIdx RegIdx) override;
private:
- struct RollbackInfo {
+ struct DeadReg {
+ /// Register index.
+ RegisterIdx Idx;
/// Original register.
Register DefReg;
- /// Original defining region.
- unsigned DefRegion;
- /// Original dependencies.
- SmallVector<Rematerializer::Reg::Dependency, 2> Dependencies;
- /// Position to re-create the register before in case of rollback. This
- /// becomes invalid if it originally points to an MI that is deleted later
- /// as a consequence of other rematerializations. In such cases \ref
- /// NextRegIdx is guaranteed to be an actual register index from which the
- /// rollback logic will determine a valid insert position before which to
- /// re-create this register.
- MachineBasicBlock::iterator InsertPos;
- /// If \ref InsertPos points to an MI defining a rematerializable register,
- /// stores its index. Otherwise equals \ref Rematerializer::NoReg.
- RegisterIdx NextRegIdx;
-
- LLVM_ABI RollbackInfo(const Rematerializer &Remater, RegisterIdx RegIdx);
+ /// Original definition of the register. The underlying MI no longer exist
+ /// at rollback time, but may be referenced as re-creation position for
+ /// previously deleted registers.
+ MachineInstr *DefMI;
+
+ LLVM_ABI DeadReg(RegisterIdx Idx, const Rematerializer &Remater)
+ : Idx(Idx), DefReg(Remater.getReg(Idx).getDefReg()),
+ DefMI(Remater.getReg(Idx).DefMI) {}
};
+ /// An insertion position in the MIR, either a MachineInstr* to insert before
+ /// or a MachineBasicBlock* to insert at the end of.
+ using InsertBeforePos = PointerUnion<MachineInstr *, MachineBasicBlock *>;
+
/// Original registers that have been deleted, in order of deletion.
- MapVector<RegisterIdx, RollbackInfo> DeadRegs;
+ SmallVector<DeadReg> DeadRegs;
+ /// Re-creation positions for all original registers that have been deleted,
+ /// in register deletion order. A position is either a MachineInstr* that
+ /// existed in the MIR at the time the rollbacker was attached to the
+ /// rematerializer, or a MachineBasicBlock*.
+ SmallVector<InsertBeforePos> Positions;
+ /// Maps all re-creation positions that exist in \ref Positions to the indices
+ /// of elements holding that position in the vector.
+ DenseMap<InsertBeforePos, SmallDenseSet<unsigned, 1>> PosToIdx;
/// Registers which have been rematerialized (from original index to
/// rematerialized index).
DenseMap<RegisterIdx, Rematerializer::RematsOf> Rematerializations;
/// Used to block further recording of events whenver we are actively rolling
/// back.
bool RollingBack = false;
+
+ InsertBeforePos makePos(MachineBasicBlock::iterator It,
+ MachineBasicBlock *MBB) const {
+ if (It == MBB->end())
+ return InsertBeforePos(MBB);
+ return InsertBeforePos(&*It);
+ }
+
+ /// Whether \p MI would be deleted if we were to rollback later. These are MIs
+ /// defining rematerializable registers whose creation has been recorded by
+ /// the rollbacker.
+ bool isRollbackableMI(const MachineInstr &MI,
+ const Rematerializer &Remater) const;
+
+ /// Switches all positions that point to \p MI to \p It in the \ref Positions
+ /// vector, and updates \ref PosToIdx accordingly. This is used when it
+ /// becomes known that \p MI is about to be permanently deleted from the MIR
+ /// and thus becomes an invalid re-creation position.
+ void invalidatePosition(MachineInstr *MI, MachineBasicBlock::iterator It);
};
} // namespace llvm
diff --git a/llvm/lib/CodeGen/Rematerializer.cpp b/llvm/lib/CodeGen/Rematerializer.cpp
index c4edface1e27f..0c1c0d36d46c2 100644
--- a/llvm/lib/CodeGen/Rematerializer.cpp
+++ b/llvm/lib/CodeGen/Rematerializer.cpp
@@ -13,9 +13,7 @@
//===----------------------------------------------------------------------===//
#include "llvm/CodeGen/Rematerializer.h"
-#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
-#include "llvm/ADT/SetVector.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/MachineBasicBlock.h"
#include "llvm/CodeGen/MachineOperand.h"
@@ -23,6 +21,7 @@
#include "llvm/CodeGen/Register.h"
#include "llvm/CodeGen/TargetRegisterInfo.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
#include <optional>
#define DEBUG_TYPE "rematerializer"
@@ -281,23 +280,22 @@ void Rematerializer::deleteRegIfUnused(RegisterIdx RootIdx) {
// Traverse the root's dependency DAG depth-first to find the set of registers
// we can delete and a legal order to delete them in.
SmallVector<RegisterIdx, 4> DepDAG{RootIdx};
- SmallSetVector<RegisterIdx, 8> DeleteOrder;
- DeleteOrder.insert(RootIdx);
+ SmallVector<RegisterIdx, 8> DeleteOrder{RootIdx};
do {
// A deleted register's dependencies may be deletable too.
const Reg &DeleteReg = getReg(DepDAG.pop_back_val());
for (const Reg::Dependency &Dep : DeleteReg.Dependencies) {
- // All dependencies loose a user (the deleted register).
+ // All dependencies lose a user (the deleted register).
Reg &DepReg = Regs[Dep.RegIdx];
DepReg.eraseUser(DeleteReg.DefMI, DeleteReg.DefRegion);
if (DepReg.Uses.empty()) {
- DeleteOrder.insert(Dep.RegIdx);
+ DeleteOrder.push_back(Dep.RegIdx);
DepDAG.push_back(Dep.RegIdx);
}
}
} while (!DepDAG.empty());
- for (RegisterIdx RegIdx : reverse(DeleteOrder)) {
+ for (RegisterIdx RegIdx : DeleteOrder) {
Reg &DeleteReg = Regs[RegIdx];
// It is possible that the defined register we are deleting doesn't have an
@@ -322,7 +320,7 @@ void Rematerializer::deleteRegIfUnused(RegisterIdx RootIdx) {
}
void Rematerializer::deleteReg(RegisterIdx RegIdx) {
- noteRegDeleted(RegIdx);
+ noteRegWillBeDeleted(RegIdx);
Reg &DeleteReg = Regs[RegIdx];
assert(DeleteReg.DefMI && "register was already deleted");
@@ -533,17 +531,14 @@ RegisterIdx Rematerializer::rematerializeReg(
return NewRegIdx;
}
-void Rematerializer::recreateReg(
- RegisterIdx RegIdx, unsigned DefRegion,
- MachineBasicBlock::iterator InsertPos, Register DefReg,
- SmallVectorImpl<Reg::Dependency> &&Dependencies) {
+void Rematerializer::recreateReg(RegisterIdx RegIdx,
+ MachineBasicBlock::iterator InsertPos,
+ Register DefReg) {
assert(RegToIdx.contains(DefReg) && "unknown defined register");
assert(RegToIdx.at(DefReg) == RegIdx && "incorrect defined register");
assert(!getReg(RegIdx).DefMI && "register is still alive");
Reg &OriginReg = Regs[RegIdx];
- OriginReg.DefRegion = DefRegion;
- OriginReg.Dependencies = std::move(Dependencies);
// Re-establish the link between origin and rematerialization if necessary.
const bool RecreateOriginalReg = isOriginalRegister(RegIdx);
@@ -563,7 +558,8 @@ void Rematerializer::recreateReg(
}
const MachineInstr &ModelDefMI = *getReg(ModelRegIdx).DefMI;
- TII.reMaterialize(*RegionMBB[DefRegion], InsertPos, DefReg, 0, ModelDefMI);
+ TII.reMaterialize(*RegionMBB[OriginReg.DefRegion], InsertPos, DefReg, 0,
+ ModelDefMI);
OriginReg.DefMI = &*std::prev(InsertPos);
postRematerialization(ModelRegIdx, RegIdx, InsertPos);
LLVM_DEBUG(dbgs() << "** Recreated " << printID(RegIdx) << " as "
@@ -762,56 +758,74 @@ Printable Rematerializer::printUser(const MachineInstr *MI,
});
}
-Rollbacker::RollbackInfo::RollbackInfo(const Rematerializer &Remater,
- RegisterIdx RegIdx) {
- const Rematerializer::Reg &Reg = Remater.getReg(RegIdx);
- DefReg = Reg.getDefReg();
- DefRegion = Reg.DefRegion;
- Dependencies = Reg.Dependencies;
-
- InsertPos = std::next(Reg.DefMI->getIterator());
- if (InsertPos != Reg.DefMI->getParent()->end())
- NextRegIdx = Remater.getDefRegIdx(*InsertPos);
- else
- NextRegIdx = Rematerializer::NoReg;
-}
-
void Rollbacker::rematerializerNoteRegCreated(const Rematerializer &Remater,
RegisterIdx RegIdx) {
if (RollingBack)
return;
+ assert(Remater.isRematerializedRegister(RegIdx) && "only remats are created");
Rematerializations[Remater.getOriginOf(RegIdx)].insert(RegIdx);
}
-void Rollbacker::rematerializerNoteRegDeleted(const Rematerializer &Remater,
- RegisterIdx RegIdx) {
- if (RollingBack || Remater.isRematerializedRegister(RegIdx))
+void Rollbacker::rematerializerNoteRegWillBeDeleted(
+ const Rematerializer &Remater, RegisterIdx RegIdx) {
+ if (RollingBack)
+ return;
+
+ // Find a valid re-creation position after the register's definition.
+ MachineInstr *DefMI = Remater.getReg(RegIdx).DefMI;
+ MachineBasicBlock *ParentMBB = DefMI->getParent();
+ MachineBasicBlock::iterator ValidPos = std::next(DefMI->getIterator());
+ while (ValidPos != ParentMBB->end() && isRollbackableMI(*ValidPos, Remater))
+ ValidPos = std::next(ValidPos);
+
+ if (Remater.isRematerializedRegister(RegIdx)) {
+ // Rematerializations will not be re-created. Previously deleted registers
+ // that reference this register's defining instruction as their re-creation
+ // position should instead be re-created at a valid position after the
+ // deleted MI.
+ invalidatePosition(DefMI, ValidPos);
return;
- DeadRegs.try_emplace(RegIdx, Remater, RegIdx);
+ }
+
+ // Original registers can be re-created. Add a re-creation position for the
+ // definition of the rematerializable register.
+ DeadRegs.push_back(DeadReg(RegIdx, Remater));
+ const InsertBeforePos InsertPos = makePos(ValidPos, ParentMBB);
+ PosToIdx[InsertPos].insert(Positions.size());
+ Positions.push_back(InsertPos);
}
void Rollbacker::rollback(Rematerializer &Remater) {
RollingBack = true;
- // Re-create deleted registers.
- for (auto &[RegIdx, Info] : DeadRegs) {
- assert(!Remater.getReg(RegIdx).isAlive() && "register should be dead");
-
- // The MI that was originally just after the MI defining the register we
- // are trying to re-create may have been deleted. In such cases, we can
- // re-create at that MI's own insert position (and apply the same logic
- // recursively).
- MachineBasicBlock::iterator InsertPos = Info.InsertPos;
- RegisterIdx NextRegIdx = Info.NextRegIdx;
- while (NextRegIdx != Rematerializer::NoReg) {
- const auto *NextRegRollback = DeadRegs.find(NextRegIdx);
- if (NextRegRollback == DeadRegs.end())
- break;
- InsertPos = NextRegRollback->second.InsertPos;
- NextRegIdx = NextRegRollback->second.NextRegIdx;
+ // As we re-create registers, map deleted definitions to re-created ones. This
+ // allows to replace invalid re-creation positions that reference deleted
+ // definitions to valid new positions while restoring original MI order.
+ DenseMap<MachineInstr *, MachineInstr *> Replacements;
+ unsigned PositionIndex = Positions.size();
+
+ // Re-create deleted registers in reverse order of deletion. Related registers
+ // are deleted in reverse def-use order so this ensures we re-create registers
+ // in def-use order. This also ensures that re-creation positions that became
+ // invalid due to later MI deletions can be corrected as we go.
+ for (const DeadReg &Reg : reverse(DeadRegs)) {
+ assert(!Remater.getReg(Reg.Idx).isAlive() && "register should be dead");
+
+ // Determine re-creation position for the register's definition.
+ MachineBasicBlock::iterator InsertPosition;
+ InsertBeforePos Pos = Positions[--PositionIndex];
+ if (auto *MBB = dyn_cast<MachineBasicBlock *>(Pos)) {
+ InsertPosition = MBB->end();
+ } else {
+ auto *MI = cast<MachineInstr *>(Pos);
+ InsertPosition = Replacements.lookup_or(MI, MI)->getIterator();
}
- Remater.recreateReg(RegIdx, Info.DefRegion, InsertPos, Info.DefReg,
- std::move(Info.Dependencies));
+
+ Remater.recreateReg(Reg.Idx, InsertPosition, Reg.DefReg);
+
+ const Rematerializer::Reg &RecreateReg = Remater.getReg(Reg.Idx);
+ if (!Replacements.insert({Reg.DefMI, RecreateReg.DefMI}).second)
+ llvm_unreachable("duplicate deleted MI");
}
// Rollback rematerializations.
@@ -828,6 +842,38 @@ void Rollbacker::rollback(Rematerializer &Remater) {
Remater.updateLiveIntervals();
DeadRegs.clear();
+ Positions.clear();
+ PosToIdx.clear();
Rematerializations.clear();
RollingBack = false;
}
+
+bool Rollbacker::isRollbackableMI(const MachineInstr &MI,
+ const Rematerializer &Remater) const {
+ RegisterIdx RegIdx = Remater.getDefRegIdx(MI);
+ if (RegIdx == Rematerializer::NoReg ||
+ !Remater.isRematerializedRegister(RegIdx))
+ return false;
+ // It is possible that the MI defines a rematerializable register that was not
+ // recorded if the rollbacker was attached to the rematerializer after the
+ // rematerialization happened. In such cases the MI won't be rolled back.
+ auto RematsOf = Rematerializations.find(Remater.getOriginOf(RegIdx));
+ if (RematsOf == Rematerializations.end())
+ return false;
+ return RematsOf->getSecond().contains(RegIdx);
+}
+
+void Rollbacker::invalidatePosition(MachineInstr *MI,
+ MachineBasicBlock::iterator It) {
+ const InsertBeforePos MIPos = InsertBeforePos(MI),
+ NewPos = makePos(It, MI->getParent());
+ auto MIIndices = PosToIdx.find(MIPos);
+ if (MIIndices == PosToIdx.end())
+ return;
+ assert(!MIIndices->getSecond().empty() && "no index hold position");
+ for (unsigned I : MIIndices->getSecond())
+ Positions[I] = NewPos;
+ PosToIdx.try_emplace(NewPos).first->getSecond().insert_range(
+ MIIndices->getSecond());
+ PosToIdx.erase(MIPos);
+}
diff --git a/llvm/unittests/CodeGen/RematerializerTest.cpp b/llvm/unittests/CodeGen/RematerializerTest.cpp
index b37760a57580e..f3dd150e5c80b 100644
--- a/llvm/unittests/CodeGen/RematerializerTest.cpp
+++ b/llvm/unittests/CodeGen/RematerializerTest.cpp
@@ -10,6 +10,7 @@
#include "CodeGenTestBase.h"
#include "llvm/CodeGen/LiveIntervals.h"
#include "llvm/CodeGen/RegisterPressure.h"
+#include "llvm/Config/Targets.h"
#include "llvm/Support/TargetSelect.h"
using namespace llvm;
@@ -640,33 +641,137 @@ TEST_F(RematerializerTest, RollbackInvalidInsertPos) {
const unsigned MBB0 = 0, MBB1 = 1;
const RegisterIdx Cst0 = 0, Cst1 = 1, Cst2 = 2, Cst3 = 3;
- // Rematerialize %0 to MBB1, deleting the original register.
- RW->rematerializeToRegion(Cst0, MBB1, DRI);
- RW.moveMIs(MBB0, MBB1, 1);
- ASSERT_REGION_SIZES();
+ auto RematToMBB1 = [&](RegisterIdx RegIdx) -> void {
+ // Rematerialize %RegIdx to MBB1, deleting the original register.
+ RW->rematerializeToRegion(RegIdx, MBB1, DRI.clear());
+ RW.moveMIs(MBB0, MBB1, 1);
+ ASSERT_REGION_SIZES();
+ };
- // Rematerialize %1 to MBB1, deleting the original register.
- RW->rematerializeToRegion(Cst1, MBB1, DRI.clear());
- RW.moveMIs(MBB0, MBB1, 1);
- ASSERT_REGION_SIZES();
+ auto GetNextMI = [&](MachineInstr *MI) -> MachineInstr * {
+ return &*std::next(MI->getIterator());
+ };
- // Rematerialize %2 to MBB1, deleting the original register.
- RW->rematerializeToRegion(Cst2, MBB1, DRI.clear());
- RW.moveMIs(MBB0, MBB1, 1);
- ASSERT_REGION_SIZES();
+ auto RollbackAndCheckOriginalOrder = [&]() -> void {
+ // Rollback and check for correct instruction order in the original
+ // defining region. The asserts on region sizes ensure that all original
+ // registers were indeed deleted and will be re-created in the original
+ // region.
+ Rollback.rollback(*RW);
+ RW.moveMIs(MBB1, MBB0, 3);
+ ASSERT_REGION_SIZES();
+
+ MachineInstr *DefCst0 = RW->getReg(Cst0).DefMI;
+ MachineInstr *DefCst1 = RW->getReg(Cst1).DefMI;
+ MachineInstr *DefCst2 = RW->getReg(Cst2).DefMI;
+ MachineInstr *DefCst3 = RW->getReg(Cst3).DefMI;
+ EXPECT_EQ(GetNextMI(DefCst0), DefCst1);
+ EXPECT_EQ(GetNextMI(DefCst1), DefCst2);
+ EXPECT_EQ(GetNextMI(DefCst2), DefCst3);
+ };
+
+ // Test every possible rematerialization order.
+
+ RematToMBB1(Cst0);
+ RematToMBB1(Cst1);
+ RematToMBB1(Cst2);
+ RollbackAndCheckOriginalOrder();
+
+ RematToMBB1(Cst0);
+ RematToMBB1(Cst2);
+ RematToMBB1(Cst1);
+ RollbackAndCheckOriginalOrder();
+
+ RematToMBB1(Cst1);
+ RematToMBB1(Cst0);
+ RematToMBB1(Cst2);
+ RollbackAndCheckOriginalOrder();
+
+ RematToMBB1(Cst1);
+ RematToMBB1(Cst2);
+ RematToMBB1(Cst0);
+ RollbackAndCheckOriginalOrder();
+
+ RematToMBB1(Cst2);
+ RematToMBB1(Cst0);
+ RematToMBB1(Cst1);
+ RollbackAndCheckOriginalOrder();
+
+ RematToMBB1(Cst2);
+ RematToMBB1(Cst1);
+ RematToMBB1(Cst0);
+ RollbackAndCheckOriginalOrder();
+ });
+}
- // Now rollback and check for correct instruction order in the original
- // defining region.
+/// Checks that rollback re-creates MIs in the correct order when the next MI
+/// after a deleted one is a rematerialization of another MI.
+TEST_F(RematerializerTest, RollbackNextPosIsRemat) {
+ StringRef MIRBody = R"MIR(
+ bb.0:
+ %0:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 0, implicit $exec, implicit $mode
+ %1:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 1, implicit $exec, implicit $mode
+
+ bb.1:
+ %2:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 2, implicit $exec, implicit $mode
+ S_NOP 0, implicit %0
+
+ bb.2:
+ %3:vgpr_32 = nofpexcept V_CVT_I32_F64_e32 3, implicit $exec, implicit $mode
+ S_NOP 0, implicit %1
+
+ bb.3:
+ S_NOP 0, implicit %2, implicit %3
+ S_ENDPGM 0
+)MIR";
+ rematerializerTest(MIRBody, [](RematerializerWrapper &RW) {
+ Rematerializer::DependencyReuseInfo DRI;
+ Rollbacker Rollback;
+
+ const unsigned MBB1 = 1, MBB2 = 2, MBB3 = 3;
+ const RegisterIdx Cst0 = 0, Cst1 = 1, Cst2 = 2, Cst3 = 3;
+
+ MachineInstr *Nop1 = &*std::prev(RW.MF.getBlockNumbered(1)->end());
+ MachineInstr *Nop2 = &*std::prev(RW.MF.getBlockNumbered(2)->end());
+ MachineInstr *Nop3 =
+ &*std::prev(std::prev(RW.MF.getBlockNumbered(3)->end()));
+
+ auto ExpectSeq = [](MachineInstr *MI, MachineInstr *ExpectedNext) {
+ MachineInstr *ActualNext = &*std::next(MI->getIterator());
+ EXPECT_EQ(ActualNext, ExpectedNext);
+ };
+
+ // This rematerialization is created right after %2, which is later
+ // rematerialized. It is *not* recorded by the rollbacker.
+ RegisterIdx RematCst0 = RW->rematerializeToRegion(Cst0, MBB1, DRI.clear());
+ ExpectSeq(RW->getReg(Cst2).DefMI, RW->getReg(RematCst0).DefMI);
+ ExpectSeq(RW->getReg(RematCst0).DefMI, Nop1);
+
+ RW->addListener(&Rollback);
+
+ // This rematerialization is created right after %3, which is later
+ // rematerialized. It is recorded by the rollbacker.
+ RegisterIdx RematCst1 = RW->rematerializeToRegion(Cst1, MBB2, DRI.clear());
+ ExpectSeq(RW->getReg(Cst3).DefMI, RW->getReg(RematCst1).DefMI);
+ ExpectSeq(RW->getReg(RematCst1).DefMI, Nop2);
+
+ RegisterIdx RematCst2 = RW->rematerializeToRegion(Cst2, MBB3, DRI.clear());
+ RegisterIdx RematCst3 = RW->rematerializeToRegion(Cst3, MBB3, DRI.clear());
+
+ ExpectSeq(RW->getReg(RematCst2).DefMI, RW->getReg(RematCst3).DefMI);
+ ExpectSeq(RW->getReg(RematCst3).DefMI, Nop3);
+
+ // After rollback, %2 and %3 should be re-created at the beginning of their
+ // respective original region.
Rollback.rollback(*RW);
- RW.moveMIs(MBB1, MBB0, 3);
- ASSERT_REGION_SIZES();
- MachineInstr &DefCst0 = *RW->getReg(Cst0).DefMI;
- MachineInstr &DefCst1 = *RW->getReg(Cst1).DefMI;
- MachineInstr &DefCst2 = *RW->getReg(Cst2).DefMI;
- MachineInstr &DefCst3 = *RW->getReg(Cst3).DefMI;
- EXPECT_EQ(std::next(DefCst0.getIterator()), DefCst1.getIterator());
- EXPECT_EQ(std::next(DefCst1.getIterator()), DefCst2.getIterator());
- EXPECT_EQ(std::next(DefCst2.getIterator()), DefCst3.getIterator());
+ // The rematerialization of %0 was not recorded so isn't rolled back, %2 is
+ // re-created right before it.
+ ExpectSeq(RW->getReg(Cst2).DefMI, RW->getReg(RematCst0).DefMI);
+ ExpectSeq(RW->getReg(RematCst0).DefMI, Nop1);
+
+ // The rematerialization of %1 was recorded so is rolled back, %3 is
+ // re-created before the S_NOP in its region.
+ ExpectSeq(RW->getReg(Cst3).DefMI, Nop2);
});
}
More information about the llvm-commits
mailing list