[llvm-branch-commits] [llvm] [TargetInstrInfo] Fix folding inline asm operands next to tied operands (PR #229627)
Bill Wendling via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Oct 25 17:13:10 PDT 2026
https://github.com/isanbard updated https://github.com/llvm/llvm-project/pull/229627
>From 674292485d0784dca0aed5949193405831eb8cbd Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Thu, 8 Oct 2026 00:28:24 -0700
Subject: [PATCH 1/8] [TargetInstrInfo] Fix folding inline asm operands next to
tied operands
foldInlineAsmMemOperand() swapped a register operand for the target's
memory operands with MachineInstr::removeOperand(), which asserts when a
later operand is tied, because moving it would break the tie. Inline asm
lists every input after every output, so folding any operand that comes
before another tied pair asserted, e.g. an "rm" input followed by the
input of a "+r" operand, or one of two "+rm" operands. Without
assertions, the moved operands kept stale tie indices. Untie the
operands, rebuild the operand list, and re-tie the remaining pairs at
their new positions.
It also gave up when the register appears in more than one operand, e.g.
one value passed to two "rm" operands, which left the greedy allocator
unable to spill that value at all. Fold every such operand into the
stack slot.
Finally, take MayLoad from the folded operands rather than from every
read of the register: a folded def whose tied use is another virtual
register (as when the fast register allocator lowers tied operands
itself) still reads the slot, and an unrelated register read of the
same value doesn't.
Nothing in tree marks an inline asm register operand foldable yet, so
the test sets the "foldable" flag in MIR and runs the greedy allocator.
Assisted-by: Claude Opus 5.5
Co-authored-by: Nick Desaulniers <ndesaulniers at google.com>
---
llvm/lib/CodeGen/TargetInstrInfo.cpp | 117 +++++++++++++-----
.../CodeGen/X86/greedy-inline-asm-fold.mir | 108 ++++++++++++++++
2 files changed, 193 insertions(+), 32 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index ae64c0086aa017..aeec7568e7d77a 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -650,29 +650,66 @@ static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI,
return NewMI;
}
-static void foldInlineAsmMemOperand(MachineInstr *MI, unsigned OpNo, int FI,
- const TargetInstrInfo &TII) {
- // If the machine operand is tied, untie it first.
- if (MI->getOperand(OpNo).isTied()) {
- unsigned TiedTo = MI->findTiedOperandIdx(OpNo);
- MI->untieRegOperand(OpNo);
- // Intentional recursion!
- foldInlineAsmMemOperand(MI, TiedTo, FI, TII);
+/// Rewrite the register operands \p Ops of the inline asm \p MI to refer to
+/// stack slot \p FI. The use tied to a folded def is folded along with it,
+/// since the two name a single location.
+static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<unsigned> Ops,
+ int FI, const TargetInstrInfo &TII) {
+ SmallVector<MachineOperand, 5> MemOps;
+ TII.getFrameIndexOperands(MemOps, FI);
+ assert(!MemOps.empty() && "getFrameIndexOperands didn't create any operands");
+ InlineAsm::Flag MemFlag(InlineAsm::Kind::Mem, MemOps.size());
+ MemFlag.setMemConstraint(InlineAsm::ConstraintCode::m);
+
+ // Find the uses to fold along with their defs, and the ties to keep. A
+ // folded operand becomes MemOps.size() operands, moving every later one, and
+ // MachineInstr can't move a tied operand. So untie everything (untying a use
+ // also unties its def), re-add the operands from the first folded one on,
+ // and re-tie the pairs that are still registers at their new positions.
+ SmallVector<unsigned, 4> FoldOps(Ops);
+ SmallVector<std::pair<unsigned, unsigned>, 4> Ties;
+ for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
+ I != E; ++I) {
+ unsigned DefIdx;
+ if (!MI.isRegTiedToDefOperand(I, &DefIdx))
+ continue;
+
+ MI.untieRegOperand(I);
+ if (is_contained(Ops, DefIdx))
+ FoldOps.push_back(I);
+ else
+ Ties.emplace_back(DefIdx, I);
}
- SmallVector<MachineOperand, 5> NewOps;
- TII.getFrameIndexOperands(NewOps, FI);
- assert(!NewOps.empty() && "getFrameIndexOperands didn't create any operands");
- MI->removeOperand(OpNo);
- MI->insert(MI->operands_begin() + OpNo, NewOps);
+ unsigned First = *llvm::min_element(FoldOps);
+ SmallVector<MachineOperand, 16> Tail(MI.operands_begin() + First,
+ MI.operands_end());
+ while (MI.getNumOperands() > First)
+ MI.removeOperand(MI.getNumOperands() - 1);
+
+ SmallVector<unsigned, 16> NewIdx;
+ for (auto [I, MO] : enumerate(Tail)) {
+ NewIdx.push_back(MI.getNumOperands());
+ if (!is_contained(FoldOps, First + I)) {
+ MI.addOperand(MO);
+ continue;
+ }
+
+ // Only the first register after a flag can be folded, so the operand just
+ // before this one is its flag. It now describes the memory operand.
+ MachineOperand &FlagMO = MI.getOperand(MI.getNumOperands() - 1);
+ assert(InlineAsm::Flag(FlagMO.getImm()).getNumOperandRegisters() == 1 &&
+ "cannot fold one register of a multi-register operand");
+ FlagMO.setImm(MemFlag);
+ for (const MachineOperand &MemOp : MemOps)
+ MI.addOperand(MemOp);
+ }
- // Change the previous operand to a MemKind InlineAsm::Flag. The second param
- // is the per-target number of operands that represent the memory operand
- // excluding this one (MD). This includes MO.
- InlineAsm::Flag F(InlineAsm::Kind::Mem, NewOps.size());
- F.setMemConstraint(InlineAsm::ConstraintCode::m);
- MachineOperand &MD = MI->getOperand(OpNo - 1);
- MD.setImm(F);
+ auto Remap = [&](unsigned Idx) {
+ return Idx < First ? Idx : NewIdx[Idx - First];
+ };
+ for (auto [DefIdx, UseIdx] : Ties)
+ MI.tieOperands(Remap(DefIdx), Remap(UseIdx));
}
// Returns nullptr if not possible to fold.
@@ -680,29 +717,45 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
ArrayRef<unsigned> Ops, int FI,
const TargetInstrInfo &TII) {
assert(MI.isInlineAsm() && "wrong opcode");
- if (Ops.size() > 1)
- return nullptr;
- unsigned Op = Ops[0];
- assert(Op && "should never be first operand");
- assert(MI.getOperand(Op).isReg() && "shouldn't be folding non-reg operands");
- if (!MI.mayFoldInlineAsmRegOp(Op))
+ // Sanity check that there are operands to fold.
+ if (Ops.empty())
return nullptr;
- MachineInstr &NewMI = TII.duplicate(*MI.getParent(), MI.getIterator(), MI);
+ // Every operand in Ops holds the same register (e.g. one value passed to two
+ // "rm" operands), so they all fold to the same stack slot. The asm reads
+ // the slot through a use, and through a def tied to a use, which need not
+ // be the same virtual register when the allocator lowers ties itself.
+ bool Reads = false;
+ bool Writes = false;
+ for (unsigned Op : Ops) {
+ assert(Op && "should never be first operand");
+ const MachineOperand &MO = MI.getOperand(Op);
+ assert(MO.isReg() && "shouldn't be folding non-reg operands");
+
+ // A tied use is only ever folded along with its def.
+ if (MI.isRegTiedToDefOperand(Op) || !MI.mayFoldInlineAsmRegOp(Op))
+ return nullptr;
- foldInlineAsmMemOperand(&NewMI, Op, FI, TII);
+ Reads |= MO.readsReg();
+ if (MO.isDef()) {
+ Writes = true;
+ Reads |=
+ MO.isTied() && MI.getOperand(MI.findTiedOperandIdx(Op)).readsReg();
+ }
+ }
+
+ MachineInstr &NewMI = TII.duplicate(*MI.getParent(), MI.getIterator(), MI);
+ foldInlineAsmMemOperands(NewMI, Ops, FI, TII);
// Update mayload/maystore metadata, and memoperands.
- const VirtRegInfo &RI =
- AnalyzeVirtRegInBundle(MI, MI.getOperand(Op).getReg());
MachineOperand &ExtraMO = NewMI.getOperand(InlineAsm::MIOp_ExtraInfo);
MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
- if (RI.Reads) {
+ if (Reads) {
ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayLoad);
Flags |= MachineMemOperand::MOLoad;
}
- if (RI.Writes) {
+ if (Writes) {
ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayStore);
Flags |= MachineMemOperand::MOStore;
}
diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
new file mode 100644
index 00000000000000..f7e1c695ebfc48
--- /dev/null
+++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
@@ -0,0 +1,108 @@
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=greedy \
+# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+
+# The greedy register allocator spills a value used by a foldable ("rm")
+# inline asm operand by folding the operand to the value's stack slot. The
+# clobbers below leave six GPRs, and the asm needs seven registers, so one
+# value is spilled. Folding replaces an operand with the five operands of a
+# memory reference, moving every operand after it, which must keep any tied
+# operands tied.
+#
+# Each test is derived from the C function and inline asm statement in its
+# comment, where CLOBBERS is:
+#
+# "rax", "rbx", "rbp", "r10", "r11", "r12", "r13", "r14", "r15"
+
+---
+# One value passed to two "rm" operands: both operands fold to the same stack
+# slot.
+#
+# void rm_same_value_twice(long a, long b, long c, long d, long e, long f,
+# long g) {
+# asm volatile("" : : "rm"(a), "rm"(a), "r"(b), "r"(c), "r"(d), "r"(e),
+# "r"(f), "r"(g) : CLOBBERS);
+# }
+name: rm_same_value_twice
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 8, alignment: 16, isImmutable: true }
+body: |
+ bb.0:
+ liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+
+ ; CHECK-LABEL: name: rm_same_value_twice
+ ; CHECK: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.[[SLOT:[0-9]+]], 1, $noreg, 0, $noreg, mem:m, %stack.[[SLOT]], 1, $noreg, 0, $noreg, reguse:GR64,
+ %0:gr64 = COPY $rdi
+ %1:gr64 = COPY $rsi
+ %2:gr64 = COPY $rdx
+ %3:gr64 = COPY $rcx
+ %4:gr64 = COPY $r8
+ %5:gr64 = COPY $r9
+ %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %0, reguse:GR64 foldable, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse:GR64, %6, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ RET 0
+...
+---
+# Folding the "rm" input moves the input tied to the "=r" output after it.
+#
+# long rm_before_tied(long a, long b, long c, long d, long e, long f,
+# long g) {
+# asm volatile("" : "+r"(g) : "rm"(a), "r"(b), "r"(c), "r"(d), "r"(e),
+# "r"(f) : CLOBBERS);
+# return g;
+# }
+name: rm_before_tied
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 8, alignment: 16, isImmutable: true }
+body: |
+ bb.0:
+ liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+
+ ; CHECK-LABEL: name: rm_before_tied
+ ; CHECK: INLINEASM &"", sideeffect mayload attdialect, regdef:GR64, def %{{[0-9]+}}, mem:m, %stack.{{[0-9]+}}, 1, $noreg, 0, $noreg, {{.*}}, reguse tiedto:$0, %{{[0-9]+}}(tied-def 3), clobber,
+ %0:gr64 = COPY $rdi
+ %1:gr64 = COPY $rsi
+ %2:gr64 = COPY $rdx
+ %3:gr64 = COPY $rcx
+ %4:gr64 = COPY $r8
+ %5:gr64 = COPY $r9
+ %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ INLINEASM &"", sideeffect attdialect, regdef:GR64, def %6, reguse:GR64 foldable, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse tiedto:$0, %6(tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ $rax = COPY %6
+ RET 0, $rax
+...
+---
+# Two read-write "+rm" operands: folding one tied pair, which reads and writes
+# the stack slot, keeps the other pair tied.
+#
+# struct pair { long a, b; };
+#
+# struct pair two_tied_rm(long a, long b, long c, long d, long e, long f,
+# long g) {
+# asm volatile("" : "+rm"(a), "+rm"(b) : "r"(c), "r"(d), "r"(e), "r"(f),
+# "r"(g) : CLOBBERS);
+# return (struct pair){a, b};
+# }
+name: two_tied_rm
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 8, alignment: 16, isImmutable: true }
+body: |
+ bb.0:
+ liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+
+ ; CHECK-LABEL: name: two_tied_rm
+ ; CHECK: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def %{{[0-9]+}}, mem:m, %stack.[[SLOT:[0-9]+]], 1, $noreg, 0, $noreg, {{.*}}, reguse tiedto:$0, %{{[0-9]+}}(tied-def 3), mem:m, %stack.[[SLOT]], 1, $noreg, 0, $noreg, clobber,
+ %0:gr64 = COPY $rdi
+ %1:gr64 = COPY $rsi
+ %2:gr64 = COPY $rdx
+ %3:gr64 = COPY $rcx
+ %4:gr64 = COPY $r8
+ %5:gr64 = COPY $r9
+ %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ INLINEASM &"", sideeffect attdialect, regdef:GR64 foldable, def %0, regdef:GR64 foldable, def %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse:GR64, %6, reguse tiedto:$0, %0(tied-def 3), reguse tiedto:$1, %1(tied-def 5), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ $rax = COPY %0
+ $rdx = COPY %1
+ RET 0, $rax, $rdx
+...
>From 7c4a3509ebb310feefcaa7be28a577d9677c2c20 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Thu, 15 Oct 2026 17:55:29 -0700
Subject: [PATCH 2/8] [TargetInstrInfo] Address review: autogenerate
greedy-inline-asm-fold.mir checks
Generate the checks with update_mir_test_checks.py so that the spills and
reloads around each folded inline asm are verified, not just the INLINEASM
itself. Fix the "=r" comment to describe the C "+r" operand, note that which
"+rm" pair greedy folds is its own choice, and add a "+&rm" case on an
uninitialized value showing that folding a pair whose tied use is undef
stores to the slot without loading from it.
Assisted-by: Claude Opus 5.5
---
.../CodeGen/X86/greedy-inline-asm-fold.mir | 102 ++++++++++++++++--
1 file changed, 94 insertions(+), 8 deletions(-)
diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
index f7e1c695ebfc48..1dcc116f8ec11d 100644
--- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
+++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
@@ -1,3 +1,4 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=greedy \
# RUN: -verify-machineinstrs %s -o - | FileCheck %s
@@ -29,9 +30,18 @@ fixedStack:
body: |
bb.0:
liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
-
; CHECK-LABEL: name: rm_same_value_twice
- ; CHECK: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.[[SLOT:[0-9]+]], 1, $noreg, 0, $noreg, mem:m, %stack.[[SLOT]], 1, $noreg, 0, $noreg, reguse:GR64,
+ ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdx
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rcx
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $r8
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r9
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[MOV64rm]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: RET 0
%0:gr64 = COPY $rdi
%1:gr64 = COPY $rsi
%2:gr64 = COPY $rdx
@@ -43,7 +53,8 @@ body: |
RET 0
...
---
-# Folding the "rm" input moves the input tied to the "=r" output after it.
+# Folding the "rm" input moves the input that "+r"(g) ties to its output,
+# which comes after the "rm" input in the operand list.
#
# long rm_before_tied(long a, long b, long c, long d, long e, long f,
# long g) {
@@ -58,9 +69,19 @@ fixedStack:
body: |
bb.0:
liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
-
; CHECK-LABEL: name: rm_before_tied
- ; CHECK: INLINEASM &"", sideeffect mayload attdialect, regdef:GR64, def %{{[0-9]+}}, mem:m, %stack.{{[0-9]+}}, 1, $noreg, 0, $noreg, {{.*}}, reguse tiedto:$0, %{{[0-9]+}}(tied-def 3), clobber,
+ ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdx
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rcx
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $r8
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r9
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, regdef:GR64, def [[MOV64rm]], mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse tiedto:$0, [[MOV64rm]](tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: $rax = COPY [[MOV64rm]]
+ ; CHECK-NEXT: RET 0, $rax
%0:gr64 = COPY $rdi
%1:gr64 = COPY $rsi
%2:gr64 = COPY $rdx
@@ -74,7 +95,9 @@ body: |
...
---
# Two read-write "+rm" operands: folding one tied pair, which reads and writes
-# the stack slot, keeps the other pair tied.
+# the stack slot, keeps the other pair tied. Which pair is folded is up to the
+# greedy allocator's spill weights, not the asm statement; it currently folds
+# "+rm"(b), so both its output and its tied input become "m" on the same slot.
#
# struct pair { long a, b; };
#
@@ -91,9 +114,25 @@ fixedStack:
body: |
bb.0:
liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
-
; CHECK-LABEL: name: two_tied_rm
- ; CHECK: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def %{{[0-9]+}}, mem:m, %stack.[[SLOT:[0-9]+]], 1, $noreg, 0, $noreg, {{.*}}, reguse tiedto:$0, %{{[0-9]+}}(tied-def 3), mem:m, %stack.[[SLOT]], 1, $noreg, 0, $noreg, clobber,
+ ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: MOV64mr %stack.4, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.4)
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $r8 :: (store (s64) into %stack.2)
+ ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $r9 :: (store (s64) into %stack.3)
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = lr-split COPY [[COPY]]
+ ; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1)
+ ; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2)
+ ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def [[COPY1]], mem:m, %stack.4, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[MOV64rm]], reguse tiedto:$0, [[COPY1]](tied-def 3), mem:m, %stack.4, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.4)
+ ; CHECK-NEXT: $rax = COPY [[COPY1]]
+ ; CHECK-NEXT: $rdx = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4)
+ ; CHECK-NEXT: RET 0, $rax, $rdx
%0:gr64 = COPY $rdi
%1:gr64 = COPY $rsi
%2:gr64 = COPY $rdx
@@ -106,3 +145,50 @@ body: |
$rdx = COPY %1
RET 0, $rax, $rdx
...
+---
+# A read-write "+rm" operand on an uninitialized value: the tied input is
+# undef, so folding the pair writes the stack slot but doesn't read it. The
+# folded asm is "maystore" without "mayload" and has only a store memoperand.
+# The output is early-clobber so that it can't share a register with one of
+# the inputs; otherwise six GPRs would be enough and nothing would be folded.
+#
+# long tied_rm_undef(long b, long c, long d, long e, long f, long g) {
+# long a;
+# asm volatile("" : "+&rm"(a) : "r"(b), "r"(c), "r"(d), "r"(e), "r"(f),
+# "r"(g) : CLOBBERS);
+# return a;
+# }
+name: tied_rm_undef
+tracksRegLiveness: true
+registers:
+ - { id: 0, class: gr64 }
+body: |
+ bb.0:
+ liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-LABEL: name: tied_rm_undef
+ ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %stack.2)
+ ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.3)
+ ; CHECK-NEXT: MOV64mr %stack.4, 1, $noreg, 0, $noreg, $r8 :: (store (s64) into %stack.4)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $r9
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1)
+ ; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2)
+ ; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
+ ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect maystore attdialect, mem:m, %stack.5, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm]], reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[COPY]], mem:m, %stack.5, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.5)
+ ; CHECK-NEXT: $rax = MOV64rm %stack.5, 1, $noreg, 0, $noreg :: (load (s64) from %stack.5)
+ ; CHECK-NEXT: RET 0, $rax
+ %1:gr64 = COPY $rdi
+ %2:gr64 = COPY $rsi
+ %3:gr64 = COPY $rdx
+ %4:gr64 = COPY $rcx
+ %5:gr64 = COPY $r8
+ %6:gr64 = COPY $r9
+ INLINEASM &"", sideeffect attdialect, regdef-ec:GR64 foldable, def early-clobber %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse:GR64, %6, reguse tiedto:$0, undef %0(tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ $rax = COPY %0
+ RET 0, $rax
+...
>From 7d7f2cdb5f61c501662d44dfa101bc4cb415171b Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Thu, 22 Oct 2026 01:53:58 -0700
Subject: [PATCH 3/8] [TargetInstrInfo] Address review: don't fold one register
of a multi-register operand
A memory operand replaces exactly one register, so return nullptr for a
folded operand whose flag holds more than one, rather than relying on an
assert that release builds drop. Also assert that every folded operand
holds the same register, and say that a caller folding a def whose tied use
is another register must store that register's value into the slot.
Test both the new bail-out and folding a rematerialized load into two "rm"
operands, which takes the fold-a-load path.
Assisted-by: Claude Opus 5.5
---
llvm/lib/CodeGen/TargetInstrInfo.cpp | 12 +++-
.../CodeGen/X86/greedy-inline-asm-fold.mir | 72 +++++++++++++++++++
2 files changed, 83 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index aeec7568e7d77a..30f64a5d8c6b5c 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -725,18 +725,28 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
// Every operand in Ops holds the same register (e.g. one value passed to two
// "rm" operands), so they all fold to the same stack slot. The asm reads
// the slot through a use, and through a def tied to a use, which need not
- // be the same virtual register when the allocator lowers ties itself.
+ // be the same virtual register when the allocator lowers ties itself. In
+ // that case the caller must store the tied use's value into the slot before
+ // the asm, as RegAllocFast does.
bool Reads = false;
bool Writes = false;
for (unsigned Op : Ops) {
assert(Op && "should never be first operand");
const MachineOperand &MO = MI.getOperand(Op);
assert(MO.isReg() && "shouldn't be folding non-reg operands");
+ assert(MO.getReg() == MI.getOperand(Ops[0]).getReg() &&
+ "folded operands must hold the same register");
// A tied use is only ever folded along with its def.
if (MI.isRegTiedToDefOperand(Op) || !MI.mayFoldInlineAsmRegOp(Op))
return nullptr;
+ // A memory operand replaces exactly one register, so one register of a
+ // multi-register operand (e.g. an i128 in two GR64s) can't be folded.
+ if (InlineAsm::Flag(MI.getOperand(Op - 1).getImm())
+ .getNumOperandRegisters() != 1)
+ return nullptr;
+
Reads |= MO.readsReg();
if (MO.isDef()) {
Writes = true;
diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
index 1dcc116f8ec11d..a89b9791e6f93e 100644
--- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
+++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
@@ -192,3 +192,75 @@ body: |
$rax = COPY %0
RET 0, $rax
...
+---
+# One register of a two-register operand can't be folded, since a memory
+# operand replaces exactly one register. The first asm clobbers every GPR, so
+# the value is spilled across it; the second asm then gets a reload instead of
+# a fold. The flag is written as an immediate, since a named flag always has
+# one register: it is "reguse:GR64 foldable" for two registers. Nothing in
+# tree marks such an operand foldable (an i128 "rm" operand picks memory), so
+# this only guards the fold.
+#
+# void rm_two_registers(__int128 a) {
+# asm volatile("" : : : ALL_GPRS);
+# asm volatile("" : : "rm"(a));
+# }
+name: rm_two_registers
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rdi, $rsi
+ ; CHECK-LABEL: name: rm_two_registers
+ ; CHECK: liveins: $rdi, $rsi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, [[MOV64rm]], [[MOV64rm1]]
+ ; CHECK-NEXT: RET 0
+ %0:gr64 = COPY $rdi
+ %1:gr64 = COPY $rsi
+ INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ INLINEASM &"", sideeffect attdialect, 1078525969, %0, %1
+ RET 0
+...
+---
+# A value loaded from an immutable stack slot is rematerialized rather than
+# spilled, by folding the load into its users. When it's passed to two "rm"
+# operands, both fold to the load's stack slot.
+#
+# void rm_load_twice(long a, long b, long c, long d, long e, long f,
+# long g) {
+# asm volatile("" : : "rm"(g), "rm"(g), "r"(a), "r"(b), "r"(c), "r"(d),
+# "r"(e), "r"(f) : CLOBBERS);
+# }
+name: rm_load_twice
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 8, alignment: 16, isImmutable: true }
+body: |
+ bb.0:
+ liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-LABEL: name: rm_load_twice
+ ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rdx
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $rcx
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r8
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY $r9
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %fixed-stack.0, 1, $noreg, 0, $noreg, mem:m, %fixed-stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[COPY5]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %fixed-stack.0, align 16)
+ ; CHECK-NEXT: RET 0
+ %0:gr64 = COPY $rdi
+ %1:gr64 = COPY $rsi
+ %2:gr64 = COPY $rdx
+ %3:gr64 = COPY $rcx
+ %4:gr64 = COPY $r8
+ %5:gr64 = COPY $r9
+ %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %6, reguse:GR64 foldable, %6, reguse:GR64, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ RET 0
+...
>From d232ab817519c142cd1724f793f2a62a492fc4b3 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Fri, 23 Oct 2026 18:35:04 -0700
Subject: [PATCH 4/8] Update llvm/lib/CodeGen/TargetInstrInfo.cpp
Co-authored-by: Nick Desaulniers <ndesaulniers at google.com>
---
llvm/lib/CodeGen/TargetInstrInfo.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 30f64a5d8c6b5c..463a887e3be87b 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -727,7 +727,7 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
// the slot through a use, and through a def tied to a use, which need not
// be the same virtual register when the allocator lowers ties itself. In
// that case the caller must store the tied use's value into the slot before
- // the asm, as RegAllocFast does.
+ // the asm.
bool Reads = false;
bool Writes = false;
for (unsigned Op : Ops) {
>From b0e54933125140d0f234176063d9efc68f8d5cd1 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Fri, 23 Oct 2026 18:55:17 -0700
Subject: [PATCH 5/8] Move folding register checks into mayFoldInlineAsmRegOp
so that it's available everywhere.
---
llvm/lib/CodeGen/MachineInstr.cpp | 9 +++++++++
llvm/lib/CodeGen/TargetInstrInfo.cpp | 8 +-------
2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 4978967c7cb112..32963d1e357b3a 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -2786,13 +2786,22 @@ bool MachineInstr::mayFoldInlineAsmRegOp(unsigned OpId) const {
if (!getOperand(OpId).isReg())
return false;
+ if (isRegTiedToDefOperand(OpId))
+ return false;
+
const MachineOperand &MD = getOperand(OpId - 1);
if (!MD.isImm())
return false;
+ // A memory operand replaces exactly one register, so one register of a
+ // multi-register operand (e.g. an i128 in two GR64s) can't be folded.
InlineAsm::Flag F(MD.getImm());
+ if (F.getNumOperandRegisters() != 1)
+ return false;
+
if (F.isRegUseKind() || F.isRegDefKind() || F.isRegDefEarlyClobberKind())
return F.getRegMayBeFolded();
+
return false;
}
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 463a887e3be87b..4418e3fc7a8248 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -738,13 +738,7 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
"folded operands must hold the same register");
// A tied use is only ever folded along with its def.
- if (MI.isRegTiedToDefOperand(Op) || !MI.mayFoldInlineAsmRegOp(Op))
- return nullptr;
-
- // A memory operand replaces exactly one register, so one register of a
- // multi-register operand (e.g. an i128 in two GR64s) can't be folded.
- if (InlineAsm::Flag(MI.getOperand(Op - 1).getImm())
- .getNumOperandRegisters() != 1)
+ if (!MI.mayFoldInlineAsmRegOp(Op))
return nullptr;
Reads |= MO.readsReg();
>From 9e860ba4cd2fa7fc247851653b40aeb23de0672c Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Fri, 23 Oct 2026 19:06:09 -0700
Subject: [PATCH 6/8] [TargetInstrInfo] Address review: don't fold a
subregister that isn't at the start of the slot
A folded operand refers to the start of the stack slot, which holds the
whole register, so a subregister is only there if it's at bit offset 0 on a
little-endian target. Folding x86's sub_8bit_hi read byte 0 instead of byte
1. Folding several operands at once made this worse: two operands holding
different subregisters of one value (e.g. "Qm"(lo), "Qm"(hi)) both read
byte 0.
Check the subregister in mayFoldInlineAsmRegOp, so the spill weights agree
with what folds. A tied def is covered too, since its tied use has the same
subregister.
Assisted-by: Claude Opus 5.5
---
llvm/include/llvm/CodeGen/MachineInstr.h | 3 +-
llvm/lib/CodeGen/MachineInstr.cpp | 11 ++
.../CodeGen/X86/greedy-inline-asm-fold.mir | 112 ++++++++++++++++++
3 files changed, 125 insertions(+), 1 deletion(-)
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index f66dff6e5bb3b1..731b92f555f4c8 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -1440,7 +1440,8 @@ class MachineInstr
}
/// Returns true if the register operand can be folded with a load or store
/// into a frame index. Does so by checking the InlineAsm::Flag immediate
- /// operand at OpId - 1.
+ /// operand at OpId - 1, and that the register (or subregister) starts at the
+ /// beginning of the stack slot.
LLVM_ABI bool mayFoldInlineAsmRegOp(unsigned OpId) const;
LLVM_ABI bool isStackAligningInlineAsm() const;
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 32963d1e357b3a..e43c5a5fa50998 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -2799,6 +2799,17 @@ bool MachineInstr::mayFoldInlineAsmRegOp(unsigned OpId) const {
if (F.getNumOperandRegisters() != 1)
return false;
+ // A memory operand refers to the start of the stack slot, which holds the
+ // whole register. A subregister is found there only if it's at bit offset 0
+ // on a little-endian target; e.g. x86's sub_8bit_hi is at byte 1. (The use
+ // tied to a def, which is folded along with it, has the def's subregister.)
+ if (unsigned SubReg = getOperand(OpId).getSubReg()) {
+ const MachineFunction &MF = *getMF();
+ if (!MF.getDataLayout().isLittleEndian() ||
+ MF.getSubtarget().getRegisterInfo()->getSubRegIdxOffset(SubReg) != 0)
+ return false;
+ }
+
if (F.isRegUseKind() || F.isRegDefKind() || F.isRegDefEarlyClobberKind())
return F.getRegMayBeFolded();
diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
index a89b9791e6f93e..9113acd3058aa1 100644
--- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
+++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
@@ -227,6 +227,118 @@ body: |
RET 0
...
---
+# A stack slot holds the whole register, so a subregister operand can be
+# folded only if the subregister starts at the beginning of the slot. Each of
+# these tests spills a value across an asm that clobbers every GPR, as above,
+# and passes its low byte, its second byte, or both to a "Qm" operand of a
+# second asm.
+#
+# The low byte (sub_8bit) is at the start of the slot, so it's folded.
+#
+# void qm_low_byte(unsigned long a) {
+# asm volatile("" : : : ALL_GPRS);
+# asm volatile("" : : "Qm"((unsigned char)a));
+# }
+name: qm_low_byte
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rdi
+ ; CHECK-LABEL: name: qm_low_byte
+ ; CHECK: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: RET 0
+ %0:gr64_abcd = COPY $rdi
+ INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_L foldable, %0.sub_8bit
+ RET 0
+...
+---
+# The second byte (sub_8bit_hi) is at byte 1 of the slot, so it gets a reload
+# instead of a fold to the start of the slot.
+#
+# void qm_high_byte(unsigned long a) {
+# asm volatile("" : : : ALL_GPRS);
+# asm volatile("" : : "Qm"((unsigned char)(a >> 8)));
+# }
+name: qm_high_byte
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rdi
+ ; CHECK-LABEL: name: qm_high_byte
+ ; CHECK: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_H foldable, [[MOV64rm]].sub_8bit_hi
+ ; CHECK-NEXT: RET 0
+ %0:gr64_abcd = COPY $rdi
+ INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_H foldable, %0.sub_8bit_hi
+ RET 0
+...
+---
+# Both bytes are subregisters of one value. Folding both operands would read
+# byte 0 for each, so neither is folded.
+#
+# void qm_both_bytes(unsigned long a) {
+# asm volatile("" : : : ALL_GPRS);
+# asm volatile("" : : "Qm"((unsigned char)a),
+# "Qm"((unsigned char)(a >> 8)));
+# }
+name: qm_both_bytes
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rdi
+ ; CHECK-LABEL: name: qm_both_bytes
+ ; CHECK: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_L foldable, [[MOV64rm]].sub_8bit, reguse:GR8_ABCD_H foldable, [[MOV64rm]].sub_8bit_hi
+ ; CHECK-NEXT: RET 0
+ %0:gr64_abcd = COPY $rdi
+ INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_L foldable, %0.sub_8bit, reguse:GR8_ABCD_H foldable, %0.sub_8bit_hi
+ RET 0
+...
+---
+# A def is folded along with the use tied to it, so neither is folded when the
+# tied use is the second byte.
+#
+# unsigned long qm_tied_high_byte(unsigned long a) {
+# asm volatile("" : : : ALL_GPRS);
+# asm volatile("" : "+Qm"(((unsigned char *)&a)[1]));
+# return a;
+# }
+name: qm_tied_high_byte
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rdi
+ ; CHECK-LABEL: name: qm_tied_high_byte
+ ; CHECK: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, regdef:GR8_ABCD_H foldable, def [[MOV64rm]].sub_8bit_hi, reguse tiedto:$0, [[MOV64rm]].sub_8bit_hi(tied-def 3)
+ ; CHECK-NEXT: $rax = COPY [[MOV64rm]]
+ ; CHECK-NEXT: RET 0, $rax
+ %0:gr64_abcd = COPY $rdi
+ INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ INLINEASM &"", sideeffect attdialect, regdef:GR8_ABCD_H foldable, def %0.sub_8bit_hi, reguse tiedto:$0, %0.sub_8bit_hi(tied-def 3)
+ $rax = COPY %0
+ RET 0, $rax
+...
+---
# A value loaded from an immutable stack slot is rematerialized rather than
# spilled, by folding the load into its users. When it's passed to two "rm"
# operands, both fold to the load's stack slot.
>From a7a659b7f7f93e9db8a32938ab10e9849952187f Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Fri, 23 Oct 2026 19:38:18 -0700
Subject: [PATCH 7/8] [TargetInstrInfo] Address review: write the two-register
flag symbolically
Now that the operand count of an inline asm flag can be written in MIR, use
"reguse:GR64 foldable ops:2" instead of an immediate whose register class ID
changes whenever a register class is added. Regenerate the checks, which
now print the five operands of each x86 memory reference as "ops:5".
Assisted-by: Claude Opus 5.5
---
.../CodeGen/X86/greedy-inline-asm-fold.mir | 22 +++++++++----------
1 file changed, 10 insertions(+), 12 deletions(-)
diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
index 9113acd3058aa1..eceb4d4d4bc37c 100644
--- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
+++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
@@ -40,7 +40,7 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $r8
; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r9
; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
- ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg, mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[MOV64rm]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[MOV64rm]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0)
; CHECK-NEXT: RET 0
%0:gr64 = COPY $rdi
%1:gr64 = COPY $rsi
@@ -79,7 +79,7 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $r8
; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r9
; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
- ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, regdef:GR64, def [[MOV64rm]], mem:m, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse tiedto:$0, [[MOV64rm]](tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, regdef:GR64, def [[MOV64rm]], mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse tiedto:$0, [[MOV64rm]](tied-def 3), clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0)
; CHECK-NEXT: $rax = COPY [[MOV64rm]]
; CHECK-NEXT: RET 0, $rax
%0:gr64 = COPY $rdi
@@ -129,7 +129,7 @@ body: |
; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1)
; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2)
; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
- ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def [[COPY1]], mem:m, %stack.4, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[MOV64rm]], reguse tiedto:$0, [[COPY1]](tied-def 3), mem:m, %stack.4, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.4)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def [[COPY1]], mem:m ops:5, %stack.4, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[MOV64rm]], reguse tiedto:$0, [[COPY1]](tied-def 3), mem:m ops:5, %stack.4, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.4)
; CHECK-NEXT: $rax = COPY [[COPY1]]
; CHECK-NEXT: $rdx = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4)
; CHECK-NEXT: RET 0, $rax, $rdx
@@ -179,7 +179,7 @@ body: |
; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2)
; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4)
- ; CHECK-NEXT: INLINEASM &"", sideeffect maystore attdialect, mem:m, %stack.5, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm]], reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[COPY]], mem:m, %stack.5, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.5)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect maystore attdialect, mem:m ops:5, %stack.5, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm]], reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[COPY]], mem:m ops:5, %stack.5, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.5)
; CHECK-NEXT: $rax = MOV64rm %stack.5, 1, $noreg, 0, $noreg :: (load (s64) from %stack.5)
; CHECK-NEXT: RET 0, $rax
%1:gr64 = COPY $rdi
@@ -196,10 +196,8 @@ body: |
# One register of a two-register operand can't be folded, since a memory
# operand replaces exactly one register. The first asm clobbers every GPR, so
# the value is spilled across it; the second asm then gets a reload instead of
-# a fold. The flag is written as an immediate, since a named flag always has
-# one register: it is "reguse:GR64 foldable" for two registers. Nothing in
-# tree marks such an operand foldable (an i128 "rm" operand picks memory), so
-# this only guards the fold.
+# a fold. Nothing in tree marks such an operand foldable (an i128 "rm" operand
+# picks memory), so this only guards the fold.
#
# void rm_two_registers(__int128 a) {
# asm volatile("" : : : ALL_GPRS);
@@ -218,12 +216,12 @@ body: |
; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1)
- ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, [[MOV64rm]], [[MOV64rm1]]
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable ops:2, [[MOV64rm]], [[MOV64rm1]]
; CHECK-NEXT: RET 0
%0:gr64 = COPY $rdi
%1:gr64 = COPY $rsi
INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
- INLINEASM &"", sideeffect attdialect, 1078525969, %0, %1
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable ops:2, %0, %1
RET 0
...
---
@@ -249,7 +247,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
- ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: RET 0
%0:gr64_abcd = COPY $rdi
INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
@@ -364,7 +362,7 @@ body: |
; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $rcx
; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r8
; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY $r9
- ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m, %fixed-stack.0, 1, $noreg, 0, $noreg, mem:m, %fixed-stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[COPY5]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %fixed-stack.0, align 16)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %fixed-stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %fixed-stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[COPY5]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %fixed-stack.0, align 16)
; CHECK-NEXT: RET 0
%0:gr64 = COPY $rdi
%1:gr64 = COPY $rsi
>From 1d50fec74e601c10811a0d58c37a392b2a1ba146 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Sun, 25 Oct 2026 10:25:01 -0700
Subject: [PATCH 8/8] [CodeGen] Fix more hazards in folding inline asm register
operands
Nothing in tree marks an inline asm operand foldable until #229632, so
these were found by a review of the stack rather than by tests:
- The spill weights took a register to be foldable into an asm if any one
of its operands there was foldable, while the folder needs all of them
to be. One value passed to an "rm" and an "r" operand could then get
spilled, not folded, and reloaded into a register the asm didn't have.
Add MachineInstr::mayFoldInlineAsmReg(), true if every operand of a
register in an asm can be folded, and use it in the folder,
CalcSpillWeights, and InlineSpiller.
- The folder now folds every operand of the register, including an undef
use that InlineSpiller leaves out of the operands it passes, which was
left behind still referring to the spilled register.
- InlineSpiller rematerialized a value passed to a foldable operand (e.g.
a constant) into a register right before the asm, which the asm may not
have had. It's spilled and folded instead.
- An asm that can fold a register no longer adds to the register's spill
weight, so the allocator spills that register before the asm's other
operands, which need registers.
- A subregister is folded at its offset in the stack slot, from
getStackSlotRange(), so getFrameIndexOperands() now takes an offset.
Before, a subregister was only folded at offset 0, but coalescing can
turn an i686 "rm" byte operand into sub_8bit_hi of a GR32.
- Folding gave the asm a single memoperand, for the stack slot, which said
that it accessed no other memory even when it had an "m" operand or a
"memory" clobber. It now gets one for any memory as well.
- Folding a load into an asm that writes the loaded slot first (through
an early-clobber "=&m" output) made the asm read what it wrote, so a
load isn't folded into an asm that stores, unless the slot is invariant.
Nor is a load that doesn't fill the register.
Also define ALL_GPRS in greedy-inline-asm-fold.mir, make qm_tied_high_byte
reach the folder, and drop -verify-machineinstrs, which llc runs anyway.
Assisted-by: Claude Opus 5.5
---
llvm/include/llvm/CodeGen/MachineInstr.h | 11 +-
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 5 +-
llvm/lib/CodeGen/CalcSpillWeights.cpp | 18 +-
llvm/lib/CodeGen/InlineSpiller.cpp | 6 +
llvm/lib/CodeGen/MachineInstr.cpp | 56 ++--
llvm/lib/CodeGen/TargetInstrInfo.cpp | 151 +++++++----
llvm/lib/Target/X86/X86InstrInfo.cpp | 3 +-
llvm/lib/Target/X86/X86InstrInfo.h | 4 +-
.../CodeGen/X86/greedy-inline-asm-fold.mir | 245 ++++++++++++++----
.../CodeGen/X86/peephole-inline-asm-fold.mir | 100 +++++++
10 files changed, 470 insertions(+), 129 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index 731b92f555f4c8..4185e8739d659b 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -1440,10 +1440,17 @@ class MachineInstr
}
/// Returns true if the register operand can be folded with a load or store
/// into a frame index. Does so by checking the InlineAsm::Flag immediate
- /// operand at OpId - 1, and that the register (or subregister) starts at the
- /// beginning of the stack slot.
+ /// operand at OpId - 1, and that the register (or subregister) has a place
+ /// in the register's stack slot.
LLVM_ABI bool mayFoldInlineAsmRegOp(unsigned OpId) const;
+ /// Returns true if \p Reg can be folded to its stack slot in this inline
+ /// asm: the asm reads or writes Reg, and every operand of Reg can be folded
+ /// (see mayFoldInlineAsmRegOp()). A use tied to a def is folded along with
+ /// the def, so it isn't checked on its own. The register allocator then
+ /// needs no register for Reg here.
+ LLVM_ABI bool mayFoldInlineAsmReg(Register Reg) const;
+
LLVM_ABI bool isStackAligningInlineAsm() const;
LLVM_ABI InlineAsm::AsmDialect getInlineAsmDialect() const;
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 97baf780275ee3..9bd88d716f4175 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -2418,13 +2418,14 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
// Get the call frame size just before MI.
unsigned getCallFrameSizeAt(MachineInstr &MI) const;
- /// Fills in the necessary MachineOperands to refer to a frame index.
+ /// Fills in the necessary MachineOperands to refer to a frame index, plus
+ /// \p Offset bytes.
/// The best way to understand this is to print `asm(""::"m"(x));` after
/// finalize-isel. Example:
/// INLINEASM ... 262190 /* mem:m */, %stack.0.x.addr, 1, $noreg, 0, $noreg
/// we would add placeholders for: ^ ^ ^ ^
virtual void getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops,
- int FI) const {
+ int FI, int64_t Offset) const {
llvm_unreachable("unknown number of operands necessary");
}
diff --git a/llvm/lib/CodeGen/CalcSpillWeights.cpp b/llvm/lib/CodeGen/CalcSpillWeights.cpp
index e958acdcb064ab..81bf22243b06ec 100644
--- a/llvm/lib/CodeGen/CalcSpillWeights.cpp
+++ b/llvm/lib/CodeGen/CalcSpillWeights.cpp
@@ -226,13 +226,17 @@ void VirtRegAuxInfo::calculateSpillWeightAndHint(LiveInterval &LI) {
LI.setWeight(Weight);
}
+/// Return true if the inline asm \p MI can access \p Reg in Reg's stack slot,
+/// which the register allocator would fold its operands to if it spills Reg.
+static bool canMemFoldInlineAsm(const MachineInstr &MI, Register Reg) {
+ return MI.isInlineAsm() && MI.mayFoldInlineAsmReg(Reg);
+}
+
static bool canMemFoldInlineAsm(LiveInterval &LI,
const MachineRegisterInfo &MRI) {
- for (const MachineOperand &MO : MRI.reg_operands(LI.reg())) {
- const MachineInstr *MI = MO.getParent();
- if (MI->isInlineAsm() && MI->mayFoldInlineAsmRegOp(MI->getOperandNo(&MO)))
+ for (const MachineInstr &MI : MRI.reg_nodbg_instructions(LI.reg()))
+ if (canMemFoldInlineAsm(MI, LI.reg()))
return true;
- }
return false;
}
@@ -315,7 +319,11 @@ float VirtRegAuxInfo::weightCalcHelper(LiveInterval &LI) {
// Force Weight onto the stack so that x86 doesn't add hidden precision.
stack_float_t Weight = 1.0f;
- if (IsSpillable) {
+ // An inline asm that can access the register in its stack slot needs no
+ // reload or spill for it, so it doesn't add to the spill weight. That makes
+ // the register a better choice to spill than the asm's other operands,
+ // which need registers.
+ if (IsSpillable && !canMemFoldInlineAsm(*MI, LI.reg())) {
// Get loop info for mi.
if (MI->getParent() != MBB) {
MBB = MI->getParent();
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index bdcf13266ae5a3..a313852443368c 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -621,6 +621,12 @@ void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) {
bool llvm::canGuaranteeAssignmentAfterRemat(Register VReg,
const MachineInstr &MI) {
+ // An inline asm that can access VReg in its stack slot (an "rm" operand)
+ // falls back on that when registers run out, which is why VReg is being
+ // spilled. A register rematerialized for it may not be available.
+ if (MI.isInlineAsm() && MI.mayFoldInlineAsmReg(VReg))
+ return false;
+
if (!RestrictStatepointRemat)
return true;
// Here's a quick explanation of the problem we're trying to handle here:
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 08150b5f69b693..ed7d88c9935b25 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -2784,37 +2784,55 @@ bool MachineInstr::mayFoldInlineAsmRegOp(unsigned OpId) const {
assert(OpId && "expected non-zero operand id");
assert(isInlineAsm() && "should only be used on inline asm");
- if (!getOperand(OpId).isReg())
- return false;
-
- if (isRegTiedToDefOperand(OpId))
+ const MachineOperand &MO = getOperand(OpId);
+ const MachineOperand &MD = getOperand(OpId - 1);
+ if (!MO.isReg() || !MD.isImm())
return false;
- const MachineOperand &MD = getOperand(OpId - 1);
- if (!MD.isImm())
+ InlineAsm::Flag F(MD.getImm());
+ if (!F.isRegUseKind() && !F.isRegDefKind() && !F.isRegDefEarlyClobberKind())
return false;
// A memory operand replaces exactly one register, so one register of a
// multi-register operand (e.g. an i128 in two GR64s) can't be folded.
- InlineAsm::Flag F(MD.getImm());
- if (F.getNumOperandRegisters() != 1)
+ if (!F.getRegMayBeFolded() || F.getNumOperandRegisters() != 1)
return false;
- // A memory operand refers to the start of the stack slot, which holds the
- // whole register. A subregister is found there only if it's at bit offset 0
- // on a little-endian target; e.g. x86's sub_8bit_hi is at byte 1. (The use
- // tied to a def, which is folded along with it, has the def's subregister.)
- if (unsigned SubReg = getOperand(OpId).getSubReg()) {
+ // A use tied to a def is folded along with the def, not on its own.
+ if (isRegTiedToDefOperand(OpId))
+ return false;
+
+ // The stack slot holds the whole register, so a subregister is folded at
+ // its offset in the slot, if it has one.
+ if (unsigned SubReg = MO.getSubReg()) {
const MachineFunction &MF = *getMF();
- if (!MF.getDataLayout().isLittleEndian() ||
- MF.getSubtarget().getRegisterInfo()->getSubRegIdxOffset(SubReg) != 0)
- return false;
+ unsigned Size, Offset;
+ return MO.getReg().isVirtual() &&
+ MF.getSubtarget().getInstrInfo()->getStackSlotRange(
+ MF.getRegInfo().getRegClass(MO.getReg()), SubReg, Size, Offset,
+ MF);
}
- if (F.isRegUseKind() || F.isRegDefKind() || F.isRegDefEarlyClobberKind())
- return F.getRegMayBeFolded();
+ return true;
+}
- return false;
+bool MachineInstr::mayFoldInlineAsmReg(Register Reg) const {
+ assert(isInlineAsm() && "should only be used on inline asm");
+
+ bool Accessed = false;
+ for (unsigned I = InlineAsm::MIOp_FirstOperand, E = getNumOperands(); I != E;
+ ++I) {
+ const MachineOperand &MO = getOperand(I);
+ if (!MO.isReg() || MO.getReg() != Reg || isRegTiedToDefOperand(I))
+ continue;
+
+ if (!mayFoldInlineAsmRegOp(I))
+ return false;
+
+ Accessed |= MO.isDef() || MO.readsReg();
+ }
+
+ return Accessed;
}
unsigned MachineInstr::removePHIIncomingValueFor(const MachineBasicBlock &MBB) {
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 08955eda50cb1e..81b61333fce48c 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -650,23 +650,17 @@ static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI,
return NewMI;
}
-/// Rewrite the register operands \p Ops of the inline asm \p MI to refer to
-/// stack slot \p FI. The use tied to a folded def is folded along with it,
-/// since the two name a single location.
-static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<unsigned> Ops,
+/// Rewrite each register operand of the inline asm \p MI that has an entry in
+/// \p Offsets (indexed by operand number) to refer to stack slot \p FI at
+/// that byte offset.
+static void foldInlineAsmMemOperands(MachineInstr &MI,
+ ArrayRef<std::optional<unsigned>> Offsets,
int FI, const TargetInstrInfo &TII) {
- SmallVector<MachineOperand, 5> MemOps;
- TII.getFrameIndexOperands(MemOps, FI);
- assert(!MemOps.empty() && "getFrameIndexOperands didn't create any operands");
- InlineAsm::Flag MemFlag(InlineAsm::Kind::Mem, MemOps.size());
- MemFlag.setMemConstraint(InlineAsm::ConstraintCode::m);
-
- // Find the uses to fold along with their defs, and the ties to keep. A
- // folded operand becomes MemOps.size() operands, moving every later one, and
- // MachineInstr can't move a tied operand. So untie everything (untying a use
- // also unties its def), re-add the operands from the first folded one on,
- // and re-tie the pairs that are still registers at their new positions.
- SmallVector<unsigned, 4> FoldOps(Ops);
+ // A folded operand becomes the operands of a memory reference, moving every
+ // later one, and MachineInstr can't move a tied operand. So untie everything
+ // (untying a use also unties its def), re-add the operands from the first
+ // folded one on, and re-tie the pairs that are still registers at their new
+ // positions.
SmallVector<std::pair<unsigned, unsigned>, 4> Ties;
for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
I != E; ++I) {
@@ -675,13 +669,14 @@ static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<unsigned> Ops,
continue;
MI.untieRegOperand(I);
- if (is_contained(Ops, DefIdx))
- FoldOps.push_back(I);
- else
+ if (!Offsets[I])
Ties.emplace_back(DefIdx, I);
}
- unsigned First = *llvm::min_element(FoldOps);
+ unsigned First = InlineAsm::MIOp_FirstOperand;
+ while (!Offsets[First])
+ ++First;
+
SmallVector<MachineOperand, 16> Tail(MI.operands_begin() + First,
MI.operands_end());
while (MI.getNumOperands() > First)
@@ -690,7 +685,8 @@ static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<unsigned> Ops,
SmallVector<unsigned, 16> NewIdx;
for (auto [I, MO] : enumerate(Tail)) {
NewIdx.push_back(MI.getNumOperands());
- if (!is_contained(FoldOps, First + I)) {
+ std::optional<unsigned> Offset = Offsets[First + I];
+ if (!Offset) {
MI.addOperand(MO);
continue;
}
@@ -700,6 +696,12 @@ static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<unsigned> Ops,
MachineOperand &FlagMO = MI.getOperand(MI.getNumOperands() - 1);
assert(InlineAsm::Flag(FlagMO.getImm()).getNumOperandRegisters() == 1 &&
"cannot fold one register of a multi-register operand");
+ SmallVector<MachineOperand, 5> MemOps;
+ TII.getFrameIndexOperands(MemOps, FI, *Offset);
+ assert(!MemOps.empty() &&
+ "getFrameIndexOperands didn't create any operands");
+ InlineAsm::Flag MemFlag(InlineAsm::Kind::Mem, MemOps.size());
+ MemFlag.setMemConstraint(InlineAsm::ConstraintCode::m);
FlagMO.setImm(MemFlag);
for (const MachineOperand &MemOp : MemOps)
MI.addOperand(MemOp);
@@ -717,40 +719,62 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
ArrayRef<unsigned> Ops, int FI,
const TargetInstrInfo &TII) {
assert(MI.isInlineAsm() && "wrong opcode");
-
- // Sanity check that there are operands to fold.
if (Ops.empty())
return nullptr;
- // Every operand in Ops holds the same register (e.g. one value passed to two
- // "rm" operands), so they all fold to the same stack slot. The asm reads
- // the slot through a use, and through a def tied to a use, which need not
- // be the same virtual register when the allocator lowers ties itself. In
- // that case the caller must store the tied use's value into the slot before
- // the asm.
- bool Reads = false;
- bool Writes = false;
+ // A tied use is only ever folded along with its def.
+ Register Reg = MI.getOperand(Ops[0]).getReg();
for (unsigned Op : Ops) {
assert(Op && "should never be first operand");
- const MachineOperand &MO = MI.getOperand(Op);
- assert(MO.isReg() && "shouldn't be folding non-reg operands");
- assert(MO.getReg() == MI.getOperand(Ops[0]).getReg() &&
+ assert(MI.getOperand(Op).isReg() && MI.getOperand(Op).getReg() == Reg &&
"folded operands must hold the same register");
-
- // A tied use is only ever folded along with its def.
- if (!MI.mayFoldInlineAsmRegOp(Op))
+ if (MI.isRegTiedToDefOperand(Op))
return nullptr;
+ }
- Reads |= MO.readsReg();
- if (MO.isDef()) {
- Writes = true;
- Reads |=
- MO.isTied() && MI.getOperand(MI.findTiedOperandIdx(Op)).readsReg();
+ // The asm can't refer to both Reg and its stack slot, so every operand of
+ // Reg is folded (e.g. one value passed to two "rm" operands), including any
+ // the caller left out, such as an undef use.
+ if (!MI.mayFoldInlineAsmReg(Reg))
+ return nullptr;
+
+ // Find the operands to fold and where each refers to in the slot, which
+ // holds the whole register: a subregister is at its offset in it. A use
+ // tied to another register's def keeps its register. The use tied to one of
+ // Reg's defs is folded along with the def, and reads the def's slot. That
+ // use need not hold Reg when the allocator lowers ties itself; the caller
+ // must then store the tied use's value into the slot before the asm.
+ MachineFunction &MF = *MI.getMF();
+ SmallVector<std::optional<unsigned>, 16> Offsets(MI.getNumOperands());
+ bool Reads = false;
+ bool Writes = false;
+ for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
+ I != E; ++I) {
+ const MachineOperand &MO = MI.getOperand(I);
+ if (!MO.isReg())
+ continue;
+
+ unsigned DefIdx;
+ const MachineOperand &SlotMO =
+ MI.isRegTiedToDefOperand(I, &DefIdx) ? MI.getOperand(DefIdx) : MO;
+ if (SlotMO.getReg() != Reg)
+ continue;
+
+ unsigned Offset = 0;
+ if (unsigned SubReg = SlotMO.getSubReg()) {
+ unsigned Size;
+ [[maybe_unused]] bool InSlot = TII.getStackSlotRange(
+ MF.getRegInfo().getRegClass(Reg), SubReg, Size, Offset, MF);
+ assert(InSlot && "foldable subregister isn't in the stack slot");
}
+
+ Offsets[I] = Offset;
+ Reads |= MO.readsReg();
+ Writes |= MO.isDef();
}
MachineInstr &NewMI = TII.duplicate(*MI.getParent(), MI.getIterator(), MI);
- foldInlineAsmMemOperands(NewMI, Ops, FI, TII);
+ foldInlineAsmMemOperands(NewMI, Offsets, FI, TII);
// Update mayload/maystore metadata, and memoperands.
MachineOperand &ExtraMO = NewMI.getOperand(InlineAsm::MIOp_ExtraInfo);
@@ -763,12 +787,27 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayStore);
Flags |= MachineMemOperand::MOStore;
}
- MachineFunction *MF = NewMI.getMF();
- const MachineFrameInfo &MFI = MF->getFrameInfo();
- MachineMemOperand *MMO = MF->getMachineMemOperand(
- MachinePointerInfo::getFixedStack(*MF, FI), Flags, MFI.getObjectSize(FI),
- MFI.getObjectAlign(FI));
- NewMI.addMemOperand(*MF, MMO);
+
+ // The memoperands of an instruction describe all of its memory accesses. An
+ // asm without any may still access memory, e.g. through an "m" operand or a
+ // "memory" clobber, so give it one for any memory as well as the slot's.
+ if (MI.memoperands_empty() && (MI.mayLoad() || MI.mayStore())) {
+ MachineMemOperand::Flags OtherFlags = MachineMemOperand::MONone;
+ if (MI.mayLoad())
+ OtherFlags |= MachineMemOperand::MOLoad;
+ if (MI.mayStore())
+ OtherFlags |= MachineMemOperand::MOStore;
+ NewMI.addMemOperand(
+ MF, MF.getMachineMemOperand(MachinePointerInfo(), OtherFlags,
+ LocationSize::beforeOrAfterPointer(),
+ Align(1)));
+ }
+
+ const MachineFrameInfo &MFI = MF.getFrameInfo();
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI), Flags,
+ MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
+ NewMI.addMemOperand(MF, MMO);
return &NewMI;
}
@@ -890,6 +929,7 @@ TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops,
// Ask the target to do the actual folding.
MachineInstr *NewMI = nullptr;
int FrameIndex = 0;
+ TypeSize MemBytes = TypeSize::getZero();
if ((MI.getOpcode() == TargetOpcode::STACKMAP ||
MI.getOpcode() == TargetOpcode::PATCHPOINT ||
@@ -899,7 +939,18 @@ TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops,
NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this);
if (NewMI)
NewMI = &*MBB.insert(MI, NewMI);
- } else if (MI.isInlineAsm() && isLoadFromStackSlot(LoadMI, FrameIndex)) {
+ } else if (MI.isInlineAsm() &&
+ isLoadFromStackSlot(LoadMI, FrameIndex, MemBytes)) {
+ // The asm reads the stack slot in place of the loaded register, so the
+ // load must fill the register, and the slot must still hold the value when
+ // the asm reads it. An asm that stores might write the slot first (e.g.
+ // through an early-clobber "=&m" output), unless it's invariant.
+ if (Ops.empty() ||
+ MemBytes * 8 != TRI.getRegSizeInBits(MI.getOperand(Ops[0]).getReg(),
+ MF.getRegInfo()) ||
+ (MI.mayStore() && !LoadMI.isDereferenceableInvariantLoad()))
+ return nullptr;
+
return foldInlineAsmMemOperand(MI, Ops, FrameIndex, *this);
} else {
// Ask the target to do the actual folding.
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 251fb43e56000b..a38d1b36576205 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -11031,10 +11031,11 @@ void X86InstrInfo::genAlternativeCodeSequence(
// See also: X86DAGToDAGISel::SelectInlineAsmMemoryOperand().
void X86InstrInfo::getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops,
- int FI) const {
+ int FI, int64_t Offset) const {
X86AddressMode M;
M.BaseType = X86AddressMode::FrameIndexBase;
M.Base.FrameIndex = FI;
+ M.Disp = Offset;
M.getFullAddress(Ops);
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index b8c1aa6b296905..cb2706ce47d59d 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -703,8 +703,8 @@ class X86InstrInfo final : public X86GenInstrInfo {
return false;
}
- void getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops,
- int FI) const override;
+ void getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops, int FI,
+ int64_t Offset) const override;
private:
/// This is a helper for convertToThreeAddress for 8 and 16-bit instructions.
diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
index eceb4d4d4bc37c..5b9c791d65e7ae 100644
--- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
+++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
@@ -1,18 +1,17 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=greedy \
-# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=greedy %s -o - \
+# RUN: | FileCheck %s
# The greedy register allocator spills a value used by a foldable ("rm")
-# inline asm operand by folding the operand to the value's stack slot. The
-# clobbers below leave six GPRs, and the asm needs seven registers, so one
-# value is spilled. Folding replaces an operand with the five operands of a
-# memory reference, moving every operand after it, which must keep any tied
-# operands tied.
+# inline asm operand by folding the operand to the value's stack slot.
+# Folding replaces an operand with the five operands of a memory reference,
+# moving every operand after it, which must keep any tied operands tied.
#
# Each test is derived from the C function and inline asm statement in its
-# comment, where CLOBBERS is:
+# comment, where CLOBBERS leaves six GPRs, and ALL_GPRS none:
#
-# "rax", "rbx", "rbp", "r10", "r11", "r12", "r13", "r14", "r15"
+# CLOBBERS: "rax", "rbx", "rbp", "r10", "r11", "r12", "r13", "r14", "r15"
+# ALL_GPRS: CLOBBERS, "rcx", "rdx", "rsi", "rdi", "r8", "r9"
---
# One value passed to two "rm" operands: both operands fold to the same stack
@@ -118,20 +117,19 @@ body: |
; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: MOV64mr %stack.4, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.4)
+ ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.3)
; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %stack.0)
; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.1)
; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $r8 :: (store (s64) into %stack.2)
- ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $r9 :: (store (s64) into %stack.3)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $r9
; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = lr-split COPY [[COPY]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = lr-split COPY [[COPY]]
; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1)
; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2)
- ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
- ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def [[COPY1]], mem:m ops:5, %stack.4, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[MOV64rm]], reguse tiedto:$0, [[COPY1]](tied-def 3), mem:m ops:5, %stack.4, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.4)
- ; CHECK-NEXT: $rax = COPY [[COPY1]]
- ; CHECK-NEXT: $rdx = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def [[COPY2]], mem:m ops:5, %stack.3, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[COPY1]], reguse:GR64, [[MOV64rm]], reguse tiedto:$0, [[COPY2]](tied-def 3), mem:m ops:5, %stack.3, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.3)
+ ; CHECK-NEXT: $rax = COPY [[COPY2]]
+ ; CHECK-NEXT: $rdx = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
; CHECK-NEXT: RET 0, $rax, $rdx
%0:gr64 = COPY $rdi
%1:gr64 = COPY $rsi
@@ -168,19 +166,14 @@ body: |
; CHECK-LABEL: name: tied_rm_undef
; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
- ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.1)
- ; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %stack.2)
- ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.3)
- ; CHECK-NEXT: MOV64mr %stack.4, 1, $noreg, 0, $noreg, $r8 :: (store (s64) into %stack.4)
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $r9
- ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
- ; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1)
- ; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2)
- ; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
- ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4)
- ; CHECK-NEXT: INLINEASM &"", sideeffect maystore attdialect, mem:m ops:5, %stack.5, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm]], reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[COPY]], mem:m ops:5, %stack.5, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.5)
- ; CHECK-NEXT: $rax = MOV64rm %stack.5, 1, $noreg, 0, $noreg :: (load (s64) from %stack.5)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rdx
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $rcx
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r8
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY $r9
+ ; CHECK-NEXT: INLINEASM &"", sideeffect maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[COPY5]], mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: RET 0, $rax
%1:gr64 = COPY $rdi
%2:gr64 = COPY $rsi
@@ -225,13 +218,12 @@ body: |
RET 0
...
---
-# A stack slot holds the whole register, so a subregister operand can be
-# folded only if the subregister starts at the beginning of the slot. Each of
-# these tests spills a value across an asm that clobbers every GPR, as above,
-# and passes its low byte, its second byte, or both to a "Qm" operand of a
-# second asm.
+# A stack slot holds the whole register, so a subregister operand is folded
+# at the subregister's offset in the slot. Each of these tests spills a value
+# across an asm that clobbers every GPR, as above, and passes its low byte,
+# its second byte, or both to a "Qm" operand of a second asm.
#
-# The low byte (sub_8bit) is at the start of the slot, so it's folded.
+# The low byte (sub_8bit) is at offset 0.
#
# void qm_low_byte(unsigned long a) {
# asm volatile("" : : : ALL_GPRS);
@@ -255,8 +247,7 @@ body: |
RET 0
...
---
-# The second byte (sub_8bit_hi) is at byte 1 of the slot, so it gets a reload
-# instead of a fold to the start of the slot.
+# The second byte (sub_8bit_hi) is at offset 1.
#
# void qm_high_byte(unsigned long a) {
# asm volatile("" : : : ALL_GPRS);
@@ -272,8 +263,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
- ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
- ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_H foldable, [[MOV64rm]].sub_8bit_hi
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 1, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: RET 0
%0:gr64_abcd = COPY $rdi
INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
@@ -281,8 +271,7 @@ body: |
RET 0
...
---
-# Both bytes are subregisters of one value. Folding both operands would read
-# byte 0 for each, so neither is folded.
+# Both bytes are subregisters of one value, each folded at its own offset.
#
# void qm_both_bytes(unsigned long a) {
# asm volatile("" : : : ALL_GPRS);
@@ -299,8 +288,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
- ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
- ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_L foldable, [[MOV64rm]].sub_8bit, reguse:GR8_ABCD_H foldable, [[MOV64rm]].sub_8bit_hi
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 1, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: RET 0
%0:gr64_abcd = COPY $rdi
INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
@@ -308,12 +296,14 @@ body: |
RET 0
...
---
-# A def is folded along with the use tied to it, so neither is folded when the
-# tied use is the second byte.
+# A def is folded along with the use tied to it, both at the offset of the
+# def's subregister. The third asm keeps the value in its slot after the
+# second.
#
# unsigned long qm_tied_high_byte(unsigned long a) {
# asm volatile("" : : : ALL_GPRS);
# asm volatile("" : "+Qm"(((unsigned char *)&a)[1]));
+# asm volatile("" : : : ALL_GPRS);
# return a;
# }
name: qm_tied_high_byte
@@ -326,13 +316,14 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
- ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
- ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, regdef:GR8_ABCD_H foldable, def [[MOV64rm]].sub_8bit_hi, reguse tiedto:$0, [[MOV64rm]].sub_8bit_hi(tied-def 3)
- ; CHECK-NEXT: $rax = COPY [[MOV64rm]]
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 1, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 1, $noreg :: (load store (s64) on %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: RET 0, $rax
%0:gr64_abcd = COPY $rdi
INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
INLINEASM &"", sideeffect attdialect, regdef:GR8_ABCD_H foldable, def %0.sub_8bit_hi, reguse tiedto:$0, %0.sub_8bit_hi(tied-def 3)
+ INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
$rax = COPY %0
RET 0, $rax
...
@@ -374,3 +365,161 @@ body: |
INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %6, reguse:GR64 foldable, %6, reguse:GR64, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
RET 0
...
+---
+# One value passed to an "rm" and an "r" operand isn't folded, since the "r"
+# operand needs it in a register anyway, so the allocator must not count on
+# spilling it: %h, which only lives across the asm, is spilled instead.
+#
+# long rm_and_r_same_value(long a, long b, long c, long d, long e, long f,
+# long g) {
+# asm volatile("" : : : ALL_GPRS);
+# long h = f + 8;
+# asm volatile("" : : "rm"(g), "r"(g), "r"(a), "r"(b), "r"(c), "r"(d),
+# "r"(e) : CLOBBERS);
+# return h;
+# }
+name: rm_and_r_same_value
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 8, alignment: 16, isImmutable: true }
+body: |
+ bb.0:
+ liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-LABEL: name: rm_and_r_same_value
+ ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %stack.2)
+ ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.3)
+ ; CHECK-NEXT: MOV64mr %stack.4, 1, $noreg, 0, $noreg, $r8 :: (store (s64) into %stack.4)
+ ; CHECK-NEXT: MOV64mr %stack.6, 1, $noreg, 0, $noreg, $r9 :: (store (s64) into %stack.6)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.6, 1, $noreg, 0, $noreg :: (load (s64) from %stack.6)
+ ; CHECK-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r [[MOV64rm]], 1, $noreg, 8, $noreg
+ ; CHECK-NEXT: MOV64mr %stack.5, 1, $noreg, 0, $noreg, [[LEA64r]] :: (store (s64) into %stack.5)
+ ; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ ; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1)
+ ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2)
+ ; CHECK-NEXT: [[MOV64rm5:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
+ ; CHECK-NEXT: [[MOV64rm6:%[0-9]+]]:gr64 = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, [[MOV64rm1]], reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[MOV64rm5]], reguse:GR64, [[MOV64rm6]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: $rax = MOV64rm %stack.5, 1, $noreg, 0, $noreg :: (load (s64) from %stack.5)
+ ; CHECK-NEXT: RET 0, $rax
+ %0:gr64 = COPY $rdi
+ %1:gr64 = COPY $rsi
+ %2:gr64 = COPY $rdx
+ %3:gr64 = COPY $rcx
+ %4:gr64 = COPY $r8
+ %5:gr64 = COPY $r9
+ INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ %7:gr64 = LEA64r %5, 1, $noreg, 8, $noreg
+ %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %6, reguse:GR64, %6, reguse:GR64, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ $rax = COPY %7
+ RET 0, $rax
+...
+---
+# An undef use of a folded register is folded along with its other operands,
+# so that none is left referring to the register. The second asm clobbers
+# every GPR, so its operands must be folded. The third asm's undef use can't
+# be folded, so nothing is, and it gets a reload. (There is no C source:
+# instruction selection doesn't emit undef inline asm operands.)
+name: rm_undef_use
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rdi
+ ; CHECK-LABEL: name: rm_undef_use
+ ; CHECK: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, [[MOV64rm]], reguse:GR64, undef [[MOV64rm]]
+ ; CHECK-NEXT: RET 0
+ %0:gr64 = COPY $rdi
+ INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %0, reguse:GR64 foldable, undef %0, reguse:GR64 foldable, %0, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %0, reguse:GR64, undef %0
+ RET 0
+...
+---
+# The memoperands of an instruction describe all of its memory accesses, so
+# the stack slot's isn't added to an asm that accesses other memory without
+# memoperands, here through an "=m" output.
+#
+# void rm_and_m(long a, long b, long c, long d, long e, long f, long g) {
+# long x;
+# asm volatile("" : "=m"(x) : "rm"(a), "r"(b), "r"(c), "r"(d), "r"(e),
+# "r"(f), "r"(g) : CLOBBERS);
+# }
+name: rm_and_m
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 8, alignment: 16, isImmutable: true }
+stack:
+ - { id: 0, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-LABEL: name: rm_and_m
+ ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdx
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rcx
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $r8
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r9
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.1, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[MOV64rm]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store unknown-size, align 1), (load (s64) from %stack.1)
+ ; CHECK-NEXT: RET 0
+ %0:gr64 = COPY $rdi
+ %1:gr64 = COPY $rsi
+ %2:gr64 = COPY $rdx
+ %3:gr64 = COPY $rcx
+ %4:gr64 = COPY $r8
+ %5:gr64 = COPY $r9
+ %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ INLINEASM &"", sideeffect maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64 foldable, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse:GR64, %6, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ RET 0
+...
+---
+# A constant passed to an "rm" operand is spilled and folded rather than
+# rematerialized into a register, which would need a register that the asm
+# doesn't leave.
+#
+# void rm_constant(long a, long b, long c, long d, long e, long f) {
+# asm volatile("" : : "rm"(42L), "r"(a), "r"(b), "r"(c), "r"(d), "r"(e),
+# "r"(f) : CLOBBERS);
+# }
+name: rm_constant
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-LABEL: name: rm_constant
+ ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rdx
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $rcx
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r8
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY $r9
+ ; CHECK-NEXT: MOV64mi32 %stack.0, 1, $noreg, 0, $noreg, 42 :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[COPY5]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: RET 0
+ %0:gr64 = COPY $rdi
+ %1:gr64 = COPY $rsi
+ %2:gr64 = COPY $rdx
+ %3:gr64 = COPY $rcx
+ %4:gr64 = COPY $r8
+ %5:gr64 = COPY $r9
+ %6:gr64 = MOV64ri32 42
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %6, reguse:GR64, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ RET 0
+...
diff --git a/llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir b/llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir
new file mode 100644
index 00000000000000..6e2d22c75ad376
--- /dev/null
+++ b/llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir
@@ -0,0 +1,100 @@
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=peephole-opt %s -o - \
+# RUN: | FileCheck %s
+
+# The peephole optimizer folds a load from a stack slot into a foldable ("rm")
+# inline asm operand that reads the loaded value, so the asm reads the slot
+# itself.
+
+--- |
+ define void @fold_load() { ret void }
+ define void @asm_may_store() { %x = alloca i64, align 8
+ ret void }
+ define void @invariant_load() { ret void }
+ define void @narrow_load() { ret void }
+ define void @undef_uses() { ret void }
+...
+---
+# CHECK-LABEL: name: fold_load
+# CHECK: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+name: fold_load
+tracksRegLiveness: true
+stack:
+ - { id: 0, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ %0:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %0, reguse:GR64 foldable, %0
+ RET 0
+...
+---
+# The asm writes x through an early-clobber output before it reads v, so v
+# can't be read from x.
+#
+# long x;
+# long v = x;
+# asm volatile("movq $0, %0; movq %1, %%rax" : "=&m"(x) : "rm"(v) : "rax");
+#
+# CHECK-LABEL: name: asm_may_store
+# CHECK: %0:gr64 = MOV64rm %stack.0.x
+# CHECK-NEXT: INLINEASM {{.*}}, reguse:GR64 foldable, %0,
+name: asm_may_store
+tracksRegLiveness: true
+stack:
+ - { id: 0, name: x, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ %0:gr64 = MOV64rm %stack.0.x, 1, $noreg, 0, $noreg :: (dereferenceable load (s64) from %ir.x)
+ INLINEASM &"movq $$0, $0\0A\09movq $1, %rax", sideeffect mayload maystore attdialect, mem:m ops:5, %stack.0.x, 1, $noreg, 0, $noreg, reguse:GR64 foldable, %0, clobber, implicit-def early-clobber $rax
+ RET 0
+...
+---
+# An asm that stores can't write an invariant slot, such as an incoming
+# argument's.
+#
+# CHECK-LABEL: name: invariant_load
+# CHECK: INLINEASM &"", sideeffect mayload maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %fixed-stack.0, 1, $noreg, 0, $noreg :: (store unknown-size, align 1), (load (s64) from %fixed-stack.0, align 16)
+name: invariant_load
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 8, alignment: 16, isImmutable: true }
+stack:
+ - { id: 0, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ %0:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ INLINEASM &"", sideeffect maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64 foldable, %0
+ RET 0
+...
+---
+# A load that fills only part of the register (here the low 4 bytes of an
+# XMM register, with the rest zeroed) isn't folded, since the asm would read
+# the whole register's width from the slot.
+#
+# CHECK-LABEL: name: narrow_load
+# CHECK: %0:vr128 = MOVSSrm %fixed-stack.0
+# CHECK-NEXT: INLINEASM &"movaps $0, %xmm1", sideeffect attdialect, reguse:VR128 foldable, %0
+name: narrow_load
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 4, alignment: 16, isImmutable: true }
+body: |
+ bb.0:
+ %0:vr128 = MOVSSrm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.0, align 16)
+ INLINEASM &"movaps $0, %xmm1", sideeffect attdialect, reguse:VR128 foldable, %0
+ RET 0
+...
+---
+# An asm that doesn't read the loaded value has nothing to fold.
+#
+# CHECK-LABEL: name: undef_uses
+# CHECK: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, undef %0, reguse:GR64 foldable, undef %0
+name: undef_uses
+tracksRegLiveness: true
+stack:
+ - { id: 0, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ %0:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, undef %0, reguse:GR64 foldable, undef %0
+ RET 0
+...
More information about the llvm-branch-commits
mailing list