[llvm] [TargetInstrInfo] enable foldMemoryOperand for InlineAsm (PR #70743)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Oct 30 15:51:28 PDT 2023
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-llvm-ir
Author: Nick Desaulniers (nickdesaulniers)
<details>
<summary>Changes</summary>
[TargetInstrInfo] enable foldMemoryOperand for InlineAsm
foldMemoryOperand looks at pairs of instructions (generally a load to
virt reg then use of the virtreg, or def of a virtreg then a store) and
attempts to combine them. This can reduce register pressure.
A prior commit added the ability to mark such a MachineOperand as
spillable. In terms of INLINEASM, this means that "rm" was used (rather
than just "r") to denote that the INLINEASM may use a memory operand
rather than a register operand. This effectively undoes decisions made
by the instruction selection framework. Callers will be added in the
register allocation frameworks. This has been tested with all of the
above (which will come as follow up patches).
Thanks to @<!-- -->topperc who suggested this at last years LLVM US Dev Meeting
and @<!-- -->qcolombet who confirmed this was the right approach.
---
Full diff: https://github.com/llvm/llvm-project/pull/70743.diff
5 Files Affected:
- (modified) llvm/include/llvm/CodeGen/MachineInstr.h (+3)
- (modified) llvm/include/llvm/CodeGen/TargetInstrInfo.h (+10)
- (modified) llvm/include/llvm/IR/InlineAsm.h (+30-5)
- (modified) llvm/lib/CodeGen/MachineInstr.cpp (+23)
- (modified) llvm/lib/CodeGen/TargetInstrInfo.cpp (+66)
``````````diff
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index 4877f43e8578d1c..93e8ff389d65673 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -1364,6 +1364,9 @@ class MachineInstr
return getOpcode() == TargetOpcode::INLINEASM ||
getOpcode() == TargetOpcode::INLINEASM_BR;
}
+ /// Returns true if the memory operand can be folded. Does so by checking the
+ /// InlineAsm::Flag immediate operand at OpId - 1.
+ bool mayFoldInlineAsmMemOp(unsigned OpId) const;
bool isStackAligningInlineAsm() const;
InlineAsm::AsmDialect getInlineAsmDialect() const;
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 8e7499ac626a747..ba98f52c27bbd2e 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -2186,6 +2186,16 @@ class TargetInstrInfo : public MCInstrInfo {
// Get the call frame size just before MI.
unsigned getCallFrameSizeAt(MachineInstr &MI) const;
+ /// Fills in the necessary MachineOperands to refer to a frame index.
+ /// The best way to understand this is to print `asm(""::"m"(x));` after
+ /// finalize-isel. Example:
+ /// INLINEASM ... 262190 /* mem:m */, %stack.0.x.addr, 1, $noreg, 0, $noreg ...
+ /// we would add placeholders for: ^ ^ ^ ^
+ virtual void
+ getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops) const {
+ llvm_unreachable("unknown number of operands necessary");
+ }
+
private:
mutable std::unique_ptr<MIRFormatter> Formatter;
unsigned CallFrameSetupOpcode, CallFrameDestroyOpcode;
diff --git a/llvm/include/llvm/IR/InlineAsm.h b/llvm/include/llvm/IR/InlineAsm.h
index 969ad42816a7e52..2d395a53608b0b7 100644
--- a/llvm/include/llvm/IR/InlineAsm.h
+++ b/llvm/include/llvm/IR/InlineAsm.h
@@ -291,18 +291,23 @@ class InlineAsm final : public Value {
// Bits 30-16 - A ConstraintCode:: value indicating the original
// constraint code. (MemConstraintCode)
// Else:
- // Bits 30-16 - The register class ID to use for the operand. (RegClass)
+ // Bits 29-16 - The register class ID to use for the operand. (RegClass)
+ // Bit 30 - If the register is permitted to be spilled.
+ // (RegMayBeSpilled)
+ // Defaults to false "r", may be set for constraints like
+ // "rm" (or "g").
//
- // As such, MatchedOperandNo, MemConstraintCode, and RegClass are views of
- // the same slice of bits, but are mutually exclusive depending on the
- // fields IsMatched then KindField.
+ // As such, MatchedOperandNo, MemConstraintCode, and
+ // (RegClass+RegMayBeSpilled) are views of the same slice of bits, but are
+ // mutually exclusive depending on the fields IsMatched then KindField.
class Flag {
uint32_t Storage;
using KindField = Bitfield::Element<Kind, 0, 3, Kind::Func>;
using NumOperands = Bitfield::Element<unsigned, 3, 13>;
using MatchedOperandNo = Bitfield::Element<unsigned, 16, 15>;
using MemConstraintCode = Bitfield::Element<ConstraintCode, 16, 15, ConstraintCode::Max>;
- using RegClass = Bitfield::Element<unsigned, 16, 15>;
+ using RegClass = Bitfield::Element<unsigned, 16, 14>;
+ using RegMayBeSpilled = Bitfield::Element<bool, 30, 1>;
using IsMatched = Bitfield::Element<bool, 31, 1>;
@@ -413,6 +418,26 @@ class InlineAsm final : public Value {
"Flag is not a memory or function constraint!");
Bitfield::set<MemConstraintCode>(Storage, ConstraintCode::Unknown);
}
+
+ /// Set a bit to denote that while this operand is some kind of register
+ /// (use, def, ...), a memory flag did appear in the original constraint
+ /// list. This is set by the instruction selection framework, and consumed
+ /// by the register allocator. While the register allocator is generally
+ /// responsible for spilling registers, we need to be able to distinguish
+ /// between registers that the register allocator has permission to spill
+ /// ("rm") vs ones it does not ("r"). This is because the inline asm may use
+ /// instructions which don't support memory addressing modes for that
+ /// operand.
+ void setRegMayBeSpilled(bool B) {
+ assert((isRegDefKind() || isRegDefEarlyClobberKind() || isRegUseKind()) &&
+ "Must be reg");
+ Bitfield::set<RegMayBeSpilled>(Storage, B);
+ }
+ bool getRegMayBeSpilled() const {
+ assert((isRegDefKind() || isRegDefEarlyClobberKind() || isRegUseKind()) &&
+ "Must be reg");
+ return Bitfield::get<RegMayBeSpilled>(Storage);
+ }
};
static std::vector<StringRef> getExtraInfoNames(unsigned ExtraInfo) {
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 048563cc2bcc4e4..92c789e85a205b4 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -1792,6 +1792,12 @@ void MachineInstr::print(raw_ostream &OS, ModuleSlotTracker &MST,
if (F.isUseOperandTiedToDef(TiedTo))
OS << " tiedto:$" << TiedTo;
+ if ((F.isRegDefKind() || F.isRegDefEarlyClobberKind() ||
+ F.isRegUseKind()) &&
+ F.getRegMayBeSpilled()) {
+ OS << " spillable";
+ }
+
OS << ']';
// Compute the index of the next operand descriptor.
@@ -2526,3 +2532,20 @@ void MachineInstr::insert(mop_iterator InsertBefore,
tieOperands(Tie1, Tie2);
}
}
+
+bool MachineInstr::mayFoldInlineAsmMemOp(unsigned OpId) const {
+ assert(OpId && "expected non-zero operand id");
+ assert(isInlineAsm() && "should only be used on inline asm");
+
+ if (!getOperand(OpId).isReg())
+ return false;
+
+ const MachineOperand &MD = getOperand(OpId - 1);
+ if (!MD.isImm())
+ return false;
+
+ InlineAsm::Flag F(MD.getImm());
+ if (F.isRegUseKind() || F.isRegDefKind() || F.isRegDefEarlyClobberKind())
+ return F.getRegMayBeSpilled();
+ return false;
+}
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index fe7efb73a2dce83..5252eeaadb2aeb2 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -565,6 +565,64 @@ static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI,
return NewMI;
}
+static void foldInlineAsmMemOperand(MachineInstr *MI, unsigned OpNo, int FI,
+ const TargetInstrInfo &TII) {
+ MachineOperand &MO = MI->getOperand(OpNo);
+ const VirtRegInfo &RI = AnalyzeVirtRegInBundle(*MI, MO.getReg());
+
+ // If the machine operand is tied, untie it first.
+ if (MO.isTied()) {
+ unsigned TiedTo = MI->findTiedOperandIdx(OpNo);
+ MI->untieRegOperand(OpNo);
+ // Intentional recursion!
+ foldInlineAsmMemOperand(MI, TiedTo, FI, TII);
+ }
+
+ // Change the operand from a register to a frame index.
+ MO.ChangeToFrameIndex(FI, MO.getTargetFlags());
+
+ SmallVector<MachineOperand, 4> NewOps;
+ TII.getFrameIndexOperands(NewOps);
+ assert(!NewOps.empty() && "getFrameIndexOperands didn't create any operands");
+ MI->insert(MI->operands_begin() + OpNo + 1, NewOps);
+
+ // Change the previous operand to a MemKind InlineAsm::Flag. The second param
+ // is the per-target number of operands that represent the memory operand
+ // excluding this one (MD). This includes MO.
+ InlineAsm::Flag F(InlineAsm::Kind::Mem, NewOps.size() + 1);
+ F.setMemConstraint(InlineAsm::ConstraintCode::m);
+ MachineOperand &MD = MI->getOperand(OpNo - 1);
+ MD.setImm(F);
+
+ // Update mayload/maystore metadata.
+ MachineOperand &ExtraMO = MI->getOperand(InlineAsm::MIOp_ExtraInfo);
+ if (RI.Reads)
+ ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayLoad);
+ if (RI.Writes)
+ ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayStore);
+}
+
+// Returns nullptr if not possible to fold.
+static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
+ ArrayRef<unsigned> Ops, int FI,
+ const TargetInstrInfo &TII) {
+ assert(MI.isInlineAsm() && "wrong opcode");
+ if (Ops.size() > 1)
+ return nullptr;
+ unsigned Op = Ops[0];
+ assert(Op && "should never be first operand");
+ assert(MI.getOperand(Op).isReg() && "shouldn't be folding non-reg operands");
+
+ if (!MI.mayFoldInlineAsmMemOp(Op))
+ return nullptr;
+
+ MachineInstr &NewMI = TII.duplicate(*MI.getParent(), MI.getIterator(), MI);
+
+ foldInlineAsmMemOperand(&NewMI, Op, FI, TII);
+
+ return &NewMI;
+}
+
MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
ArrayRef<unsigned> Ops, int FI,
LiveIntervals *LIS,
@@ -612,6 +670,8 @@ MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
NewMI = foldPatchpoint(MF, MI, Ops, FI, *this);
if (NewMI)
MBB->insert(MI, NewMI);
+ } else if (MI.isInlineAsm()) {
+ NewMI = foldInlineAsmMemOperand(MI, Ops, FI, *this);
} else {
// Ask the target to do the actual folding.
NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, FI, LIS, VRM);
@@ -683,6 +743,8 @@ MachineInstr *TargetInstrInfo::foldMemoryOperand(MachineInstr &MI,
NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this);
if (NewMI)
NewMI = &*MBB.insert(MI, NewMI);
+ } else if (MI.isInlineAsm() && isLoadFromStackSlot(LoadMI, FrameIndex)) {
+ NewMI = foldInlineAsmMemOperand(MI, Ops, FrameIndex, *this);
} else {
// Ask the target to do the actual folding.
NewMI = foldMemoryOperandImpl(MF, MI, Ops, MI, LoadMI, LIS);
@@ -1639,6 +1701,10 @@ std::string TargetInstrInfo::createMIROperandComment(
if (F.isUseOperandTiedToDef(TiedTo))
OS << " tiedto:$" << TiedTo;
+ if ((F.isRegDefKind() || F.isRegDefEarlyClobberKind() || F.isRegUseKind()) &&
+ F.getRegMayBeSpilled())
+ OS << " spillable";
+
return OS.str();
}
``````````
</details>
https://github.com/llvm/llvm-project/pull/70743
More information about the llvm-commits
mailing list