[llvm-branch-commits] [llvm] [CodeGen] Report an error for a direct inline asm output in memory (PR #229628)
Bill Wendling via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Mon Oct 26 00:33:50 PDT 2026
https://github.com/isanbard updated https://github.com/llvm/llvm-project/pull/229628
>From dc00180b5275299af9ebd6eb00cf342a46bf9cd9 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Tue, 6 Oct 2026 06:12:08 -0700
Subject: [PATCH 1/9] [CodeGen] Report an error for a direct inline asm output
in memory
An inline asm output returned by value has no memory to write to, yet a
constraint such as "=rm" picks memory, the most general constraint, as
does "=m". SelectionDAG asserted on that ("Can only indirectify direct
input operands!"), and GlobalISel dereferenced a null pointer. Clang
never emits such an output, since it passes the address of a memory
output, but other IR can. Report "cannot handle direct memory outputs
yet for constraint 'm'" instead, like the other inline asm errors there.
Assisted-by: Claude Opus 5.5
---
.../CodeGen/GlobalISel/InlineAsmLowering.cpp | 10 ++++++
.../SelectionDAG/SelectionDAGBuilder.cpp | 22 +++++++++----
.../inline-asm-direct-mem-output-error.ll | 13 ++++++++
.../X86/inline-asm-direct-mem-output-error.ll | 31 +++++++++++++++++++
4 files changed, 70 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
create mode 100644 llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
index 19eae930cc26b1..5d66cb5e332276 100644
--- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
@@ -337,6 +337,16 @@ bool InlineAsmLowering::lowerInlineAsm(
switch (OpInfo.Type) {
case InlineAsm::isOutput:
if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
+ // A direct output is the call's result, with no address to write to.
+ if (!OpInfo.isIndirect) {
+ emitInlineAsmError(MIRBuilder, Call,
+ "cannot handle direct memory outputs yet for "
+ "constraint '" +
+ Twine(OpInfo.ConstraintCode) + "'",
+ GetOrCreateVRegs(Call));
+ return true;
+ }
+
const InlineAsm::ConstraintCode ConstraintID =
TLI->getInlineAsmMemConstraint(OpInfo.ConstraintCode);
assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index b595de3734bb9f..e994b6af23eb04 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -10504,8 +10504,9 @@ constructOperandInfo(ConstraintDecisionInfo &Info,
return false;
}
-/// Compute which constraint option to use for each operand.
-static void
+/// Compute which constraint option to use for each operand. Returns true (and
+/// sets Info.ErrorMsg) on failure.
+static bool
computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
TargetLowering::AsmOperandInfoVector &TargetConstraints,
SelectionDAGBuilder &Builder, const TargetLowering &TLI,
@@ -10567,9 +10568,14 @@ computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
// need to provide an address for the memory input.
if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
!OpInfo.isIndirect) {
- assert((OpInfo.isMultipleAlternative ||
- (OpInfo.Type == InlineAsm::isInput)) &&
- "Can only indirectify direct input operands!");
+ // Only an input has a value to store to memory. A direct output, which
+ // Clang never emits with a memory constraint but other IR can, would
+ // need a stack slot to be reloaded after the asm.
+ if (!OpInfo.isMultipleAlternative && OpInfo.Type != InlineAsm::isInput) {
+ Info.ErrorMsg << "cannot handle direct memory outputs yet for "
+ << "constraint '" << OpInfo.ConstraintCode << "'";
+ return true;
+ }
// Memory operands really want the address of the value.
Info.Chain = getAddressForMemoryInput(Info.Chain, Builder.getCurSDLoc(),
@@ -10582,6 +10588,8 @@ computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
OpInfo.isIndirect = true;
}
}
+
+ return false;
}
/// Prepare DAG-level operands. As part of this, assign virtual and physical
@@ -10875,7 +10883,9 @@ determineConstraints(ConstraintDecisionInfo &Info,
Info.Chain = Builder.lowerStartEH(Info.Chain, EHPadBB, Info.BeginLabel);
// Second pass: Compute which constraint option to use.
- computeConstraintToUse(Info, Call, TargetConstraints, Builder, TLI, TM, DAG);
+ if (computeConstraintToUse(Info, Call, TargetConstraints, Builder, TLI, TM,
+ DAG))
+ return true;
// AsmNodeOperands - The operands for the ISD::INLINEASM node.
Info.AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
new file mode 100644
index 00000000000000..7ebe9f006943f9
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
@@ -0,0 +1,13 @@
+; RUN: not llc -mtriple=aarch64 -filetype=null < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=aarch64 -global-isel -global-isel-abort=1 \
+; RUN: -filetype=null < %s 2>&1 | FileCheck %s
+
+; A direct output is the asm's result, so there is no memory to write it to.
+; Picking a memory constraint for one is an error, not a crash, with either
+; instruction selector.
+
+; CHECK: error: cannot handle direct memory outputs yet for constraint 'm'
+define i64 @rm_output() {
+ %r = call i64 asm "# $0", "=rm"()
+ ret i64 %r
+}
diff --git a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
new file mode 100644
index 00000000000000..3f3a9168de38f8
--- /dev/null
+++ b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
@@ -0,0 +1,31 @@
+; RUN: not llc -mtriple=x86_64-unknown-linux-gnu -O0 -filetype=null < %s 2>&1 \
+; RUN: | FileCheck --check-prefix=O0 --implicit-check-not=error: %s
+; RUN: not llc -mtriple=x86_64-unknown-linux-gnu -filetype=null < %s 2>&1 \
+; RUN: | FileCheck --check-prefix=O2 --implicit-check-not=error: %s
+
+; A direct output is the asm's result, so there is no memory to write it to.
+; Picking a memory constraint for one is an error, not a crash.
+
+; "rm" picks memory, the most general constraint.
+; O0: error: cannot handle direct memory outputs yet for constraint 'm'
+; O2: error: cannot handle direct memory outputs yet for constraint 'm'
+define i32 @rm_i32() {
+ %r = call i32 asm "# $0", "=rm"()
+ ret i32 %r
+}
+
+; The same for a value no 'r' register can hold.
+; O0: error: cannot handle direct memory outputs yet for constraint 'm'
+; O2: error: cannot handle direct memory outputs yet for constraint 'm'
+define x86_fp80 @rm_x86_fp80() {
+ %r = call x86_fp80 asm "# $0", "=rm"()
+ ret x86_fp80 %r
+}
+
+; "=m" has no other choice.
+; O0: error: cannot handle direct memory outputs yet for constraint 'm'
+; O2: error: cannot handle direct memory outputs yet for constraint 'm'
+define i32 @m_i32() {
+ %r = call i32 asm "# $0", "=m"()
+ ret i32 %r
+}
>From 6a0c860c77098eaf551deb00912f1bfe080fb260 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Thu, 15 Oct 2026 18:20:52 -0700
Subject: [PATCH 2/9] [CodeGen] Address review: error for "=m|r" direct outputs
too
A direct output whose chosen alternative is memory, e.g. "=m|r", still
crashed SelectionDAG, which exempted multiple-alternative operands from the
check and then asked for the address of an output with no value. Drop the
exemption and test "=m|r" with both instruction selectors. Name the
selector explicitly in the RUN lines rather than relying on -O levels,
since X86 GlobalISel can't lower inline asm and both X86 lines ran
SelectionDAG anyway. Drop "yet" from the error: Clang never emits a
direct memory output, so there are no plans to support one.
Assisted-by: Claude Opus 5.5
---
.../CodeGen/GlobalISel/InlineAsmLowering.cpp | 2 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 9 +++----
.../inline-asm-direct-mem-output-error.ll | 14 ++++++++---
.../X86/inline-asm-direct-mem-output-error.ll | 24 +++++++++++--------
4 files changed, 31 insertions(+), 18 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
index 5d66cb5e332276..210f693f7bf583 100644
--- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
@@ -340,7 +340,7 @@ bool InlineAsmLowering::lowerInlineAsm(
// A direct output is the call's result, with no address to write to.
if (!OpInfo.isIndirect) {
emitInlineAsmError(MIRBuilder, Call,
- "cannot handle direct memory outputs yet for "
+ "cannot handle direct memory outputs for "
"constraint '" +
Twine(OpInfo.ConstraintCode) + "'",
GetOrCreateVRegs(Call));
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index e994b6af23eb04..c15e96738752cc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -10570,10 +10570,11 @@ computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
!OpInfo.isIndirect) {
// Only an input has a value to store to memory. A direct output, which
// Clang never emits with a memory constraint but other IR can, would
- // need a stack slot to be reloaded after the asm.
- if (!OpInfo.isMultipleAlternative && OpInfo.Type != InlineAsm::isInput) {
- Info.ErrorMsg << "cannot handle direct memory outputs yet for "
- << "constraint '" << OpInfo.ConstraintCode << "'";
+ // need a stack slot to be reloaded after the asm. That includes an
+ // output whose chosen alternative is memory, e.g. "=m|r".
+ if (OpInfo.Type != InlineAsm::isInput) {
+ Info.ErrorMsg << "cannot handle direct memory outputs for constraint '"
+ << OpInfo.ConstraintCode << "'";
return true;
}
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
index 7ebe9f006943f9..12aa4a46da9989 100644
--- a/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
+++ b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
@@ -1,13 +1,21 @@
-; RUN: not llc -mtriple=aarch64 -filetype=null < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=aarch64 -global-isel=0 -filetype=null < %s 2>&1 \
+; RUN: | FileCheck --implicit-check-not=error: %s
; RUN: not llc -mtriple=aarch64 -global-isel -global-isel-abort=1 \
-; RUN: -filetype=null < %s 2>&1 | FileCheck %s
+; RUN: -filetype=null < %s 2>&1 | FileCheck --implicit-check-not=error: %s
; A direct output is the asm's result, so there is no memory to write it to.
; Picking a memory constraint for one is an error, not a crash, with either
; instruction selector.
-; CHECK: error: cannot handle direct memory outputs yet for constraint 'm'
+; CHECK: error: cannot handle direct memory outputs for constraint 'm'
define i64 @rm_output() {
%r = call i64 asm "# $0", "=rm"()
ret i64 %r
}
+
+; With multiple alternatives, the memory one is chosen here, too.
+; CHECK: error: cannot handle direct memory outputs for constraint 'm'
+define i64 @multi_alt() {
+ %r = call i64 asm "# $0", "=m|r"()
+ ret i64 %r
+}
diff --git a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
index 3f3a9168de38f8..00135acfdc4769 100644
--- a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
@@ -1,31 +1,35 @@
-; RUN: not llc -mtriple=x86_64-unknown-linux-gnu -O0 -filetype=null < %s 2>&1 \
-; RUN: | FileCheck --check-prefix=O0 --implicit-check-not=error: %s
-; RUN: not llc -mtriple=x86_64-unknown-linux-gnu -filetype=null < %s 2>&1 \
-; RUN: | FileCheck --check-prefix=O2 --implicit-check-not=error: %s
+; X86 GlobalISel can't lower inline asm, so only SelectionDAG is tested here;
+; see the AArch64 test for both instruction selectors.
+; RUN: not llc -mtriple=x86_64-unknown-linux-gnu -global-isel=0 \
+; RUN: -filetype=null < %s 2>&1 | FileCheck --implicit-check-not=error: %s
; A direct output is the asm's result, so there is no memory to write it to.
; Picking a memory constraint for one is an error, not a crash.
; "rm" picks memory, the most general constraint.
-; O0: error: cannot handle direct memory outputs yet for constraint 'm'
-; O2: error: cannot handle direct memory outputs yet for constraint 'm'
+; CHECK: error: cannot handle direct memory outputs for constraint 'm'
define i32 @rm_i32() {
%r = call i32 asm "# $0", "=rm"()
ret i32 %r
}
; The same for a value no 'r' register can hold.
-; O0: error: cannot handle direct memory outputs yet for constraint 'm'
-; O2: error: cannot handle direct memory outputs yet for constraint 'm'
+; CHECK: error: cannot handle direct memory outputs for constraint 'm'
define x86_fp80 @rm_x86_fp80() {
%r = call x86_fp80 asm "# $0", "=rm"()
ret x86_fp80 %r
}
; "=m" has no other choice.
-; O0: error: cannot handle direct memory outputs yet for constraint 'm'
-; O2: error: cannot handle direct memory outputs yet for constraint 'm'
+; CHECK: error: cannot handle direct memory outputs for constraint 'm'
define i32 @m_i32() {
%r = call i32 asm "# $0", "=m"()
ret i32 %r
}
+
+; With multiple alternatives, the memory one is chosen here, too.
+; CHECK: error: cannot handle direct memory outputs for constraint 'm'
+define i32 @multi_alt() {
+ %r = call i32 asm "# $0", "=m|r"()
+ ret i32 %r
+}
>From f89617d6cd1d072ef27cc93e627bcb82fa0f529e Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Fri, 23 Oct 2026 21:27:20 -0700
Subject: [PATCH 3/9] [CodeGen] Address review: report a memory constraint on a
label as such
A direct label operand with a memory constraint (e.g. "!m") isn't an
input, so it was reported as a direct memory output. Give it its own
error, and update the comment above the check, which still described only
memory inputs.
Assisted-by: Claude Opus 5.5
---
.../SelectionDAG/SelectionDAGBuilder.cpp | 18 ++++++++++++------
.../X86/inline-asm-direct-mem-output-error.ll | 12 ++++++++++++
2 files changed, 24 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index c15e96738752cc..7defabd4f11ae0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -10564,14 +10564,20 @@ computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
OpInfo.ConstraintType = TargetLowering::C_Address;
}
- // If this is a memory input, and if the operand is not indirect, do what we
- // need to provide an address for the memory input.
+ // A direct operand with a memory constraint needs an address. Only an
+ // input has a value that can be stored to a stack slot to provide one. A
+ // direct output, which Clang never emits with a memory constraint but
+ // other IR can, would need a stack slot to be reloaded after the asm. That
+ // includes an output whose chosen alternative is memory, e.g. "=m|r". A
+ // label (e.g. "!m") is a branch target, which has no value in memory.
if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
!OpInfo.isIndirect) {
- // Only an input has a value to store to memory. A direct output, which
- // Clang never emits with a memory constraint but other IR can, would
- // need a stack slot to be reloaded after the asm. That includes an
- // output whose chosen alternative is memory, e.g. "=m|r".
+ if (OpInfo.Type == InlineAsm::isLabel) {
+ Info.ErrorMsg << "cannot handle memory constraint '"
+ << OpInfo.ConstraintCode << "' for a label";
+ return true;
+ }
+
if (OpInfo.Type != InlineAsm::isInput) {
Info.ErrorMsg << "cannot handle direct memory outputs for constraint '"
<< OpInfo.ConstraintCode << "'";
diff --git a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
index 00135acfdc4769..709fe787398cc0 100644
--- a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
@@ -33,3 +33,15 @@ define i32 @multi_alt() {
%r = call i32 asm "# $0", "=m|r"()
ret i32 %r
}
+
+; A label is a branch target, which isn't in memory either. It isn't an
+; output, so the error says so.
+; CHECK: error: cannot handle memory constraint 'm' for a label
+define void @label_mem() {
+ callbr void asm "# $0", "!m"()
+ to label %fallthrough [label %indirect]
+fallthrough:
+ ret void
+indirect:
+ ret void
+}
>From 7bb33241c8755130b9a3b8f7c124bf30d268c0cf Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Sun, 25 Oct 2026 10:27:45 -0700
Subject: [PATCH 4/9] [CodeGen] Report the label error for an indirect label
too
A memory constraint on an indirect label ("!*m") skipped the label check,
which only looked at direct operands, and asserted later. Check labels
first, whatever their indirection, and shorten the comment, which still
mentioned the "=m|r" exemption and a stack slot for every memory input
(a constant goes in the constant pool).
Assisted-by: Claude Opus 5.5
---
.../SelectionDAG/SelectionDAGBuilder.cpp | 23 +++++++++----------
.../X86/inline-asm-direct-mem-output-error.ll | 11 +++++++++
2 files changed, 22 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 0f9e8f2ae98fcb..caa72f819e1e79 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -10566,20 +10566,19 @@ computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
OpInfo.ConstraintType = TargetLowering::C_Address;
}
- // A direct operand with a memory constraint needs an address. Only an
- // input has a value that can be stored to a stack slot to provide one. A
- // direct output, which Clang never emits with a memory constraint but
- // other IR can, would need a stack slot to be reloaded after the asm. That
- // includes an output whose chosen alternative is memory, e.g. "=m|r". A
- // label (e.g. "!m") is a branch target, which has no value in memory.
+ // A label (e.g. "!m") is a branch target, which has no value in memory.
if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
- !OpInfo.isIndirect) {
- if (OpInfo.Type == InlineAsm::isLabel) {
- Info.ErrorMsg << "cannot handle memory constraint '"
- << OpInfo.ConstraintCode << "' for a label";
- return true;
- }
+ OpInfo.Type == InlineAsm::isLabel) {
+ Info.ErrorMsg << "cannot handle memory constraint '"
+ << OpInfo.ConstraintCode << "' for a label";
+ return true;
+ }
+ // A direct operand with a memory constraint needs an address. An input's
+ // value is put in memory for one. A direct output would have to be
+ // reloaded from memory after the asm, which isn't supported.
+ if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
+ !OpInfo.isIndirect) {
if (OpInfo.Type != InlineAsm::isInput) {
Info.ErrorMsg << "cannot handle direct memory outputs for constraint '"
<< OpInfo.ConstraintCode << "'";
diff --git a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
index 709fe787398cc0..9588a8992847d3 100644
--- a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
@@ -45,3 +45,14 @@ fallthrough:
indirect:
ret void
}
+
+; The same for an indirect label.
+; CHECK: error: cannot handle memory constraint 'm' for a label
+define void @label_indirect_mem() {
+ callbr void asm "# $0", "!*m"()
+ to label %fallthrough [label %indirect]
+fallthrough:
+ ret void
+indirect:
+ ret void
+}
>From 579870ae81f02fcc67c7bc998a486e03ab8307c4 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Mon, 26 Oct 2026 00:15:59 -0700
Subject: [PATCH 5/9] [CodeGen] Fold every operand of a register into an inline
asm
The spill weights took a register to be foldable into an asm if any one
of its operands there was foldable, while the folder needs all of them
to be. One value passed to an "rm" and an "r" operand could then get
spilled, not folded, and the asm ran out of registers. Add
MachineInstr::mayFoldInlineAsmReg(), true if every operand of a register
in an asm can be folded, and use it in the folder and CalcSpillWeights.
The folder now folds every operand of the register, including an undef
use that InlineSpiller leaves out of the operands it passes, which was
left behind still referring to the spilled register.
Nothing in tree marks an inline asm operand foldable until #229632, so
the tests set the "foldable" flag in MIR. Also define ALL_GPRS in
greedy-inline-asm-fold.mir, and drop -verify-machineinstrs, which llc
runs anyway.
Assisted-by: Claude Opus 5.5
---
llvm/include/llvm/CodeGen/MachineInstr.h | 7 ++
llvm/lib/CodeGen/CalcSpillWeights.cpp | 12 ++-
llvm/lib/CodeGen/MachineInstr.cpp | 19 ++++
llvm/lib/CodeGen/TargetInstrInfo.cpp | 85 +++++++++-------
.../CodeGen/X86/greedy-inline-asm-fold.mir | 98 +++++++++++++++++--
5 files changed, 171 insertions(+), 50 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index 731b92f555f4c8..dd689cf280f371 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -1444,6 +1444,13 @@ class MachineInstr
/// beginning of the stack slot.
LLVM_ABI bool mayFoldInlineAsmRegOp(unsigned OpId) const;
+ /// Returns true if \p Reg can be folded to its stack slot in this inline
+ /// asm: the asm reads or writes Reg, and every operand of Reg can be folded
+ /// (see mayFoldInlineAsmRegOp()). A use tied to a def is folded along with
+ /// the def, so it isn't checked on its own. The register allocator then
+ /// needs no register for Reg here.
+ LLVM_ABI bool mayFoldInlineAsmReg(Register Reg) const;
+
LLVM_ABI bool isStackAligningInlineAsm() const;
LLVM_ABI InlineAsm::AsmDialect getInlineAsmDialect() const;
diff --git a/llvm/lib/CodeGen/CalcSpillWeights.cpp b/llvm/lib/CodeGen/CalcSpillWeights.cpp
index e958acdcb064ab..26d265c2a986e8 100644
--- a/llvm/lib/CodeGen/CalcSpillWeights.cpp
+++ b/llvm/lib/CodeGen/CalcSpillWeights.cpp
@@ -226,13 +226,17 @@ void VirtRegAuxInfo::calculateSpillWeightAndHint(LiveInterval &LI) {
LI.setWeight(Weight);
}
+/// Return true if the inline asm \p MI can access \p Reg in Reg's stack slot,
+/// which the register allocator would fold its operands to if it spills Reg.
+static bool canMemFoldInlineAsm(const MachineInstr &MI, Register Reg) {
+ return MI.isInlineAsm() && MI.mayFoldInlineAsmReg(Reg);
+}
+
static bool canMemFoldInlineAsm(LiveInterval &LI,
const MachineRegisterInfo &MRI) {
- for (const MachineOperand &MO : MRI.reg_operands(LI.reg())) {
- const MachineInstr *MI = MO.getParent();
- if (MI->isInlineAsm() && MI->mayFoldInlineAsmRegOp(MI->getOperandNo(&MO)))
+ for (const MachineInstr &MI : MRI.reg_nodbg_instructions(LI.reg()))
+ if (canMemFoldInlineAsm(MI, LI.reg()))
return true;
- }
return false;
}
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 08150b5f69b693..23f5170ea66d20 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -2817,6 +2817,25 @@ bool MachineInstr::mayFoldInlineAsmRegOp(unsigned OpId) const {
return false;
}
+bool MachineInstr::mayFoldInlineAsmReg(Register Reg) const {
+ assert(isInlineAsm() && "should only be used on inline asm");
+
+ bool Accessed = false;
+ for (unsigned I = InlineAsm::MIOp_FirstOperand, E = getNumOperands(); I != E;
+ ++I) {
+ const MachineOperand &MO = getOperand(I);
+ if (!MO.isReg() || MO.getReg() != Reg || isRegTiedToDefOperand(I))
+ continue;
+
+ if (!mayFoldInlineAsmRegOp(I))
+ return false;
+
+ Accessed |= MO.isDef() || MO.readsReg();
+ }
+
+ return Accessed;
+}
+
unsigned MachineInstr::removePHIIncomingValueFor(const MachineBasicBlock &MBB) {
assert(isPHI());
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 08955eda50cb1e..47bfa9cdc94610 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -650,10 +650,9 @@ static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI,
return NewMI;
}
-/// Rewrite the register operands \p Ops of the inline asm \p MI to refer to
-/// stack slot \p FI. The use tied to a folded def is folded along with it,
-/// since the two name a single location.
-static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<unsigned> Ops,
+/// Rewrite each register operand of the inline asm \p MI that's set in \p Fold
+/// (indexed by operand number) to refer to stack slot \p FI.
+static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<bool> Fold,
int FI, const TargetInstrInfo &TII) {
SmallVector<MachineOperand, 5> MemOps;
TII.getFrameIndexOperands(MemOps, FI);
@@ -661,12 +660,11 @@ static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<unsigned> Ops,
InlineAsm::Flag MemFlag(InlineAsm::Kind::Mem, MemOps.size());
MemFlag.setMemConstraint(InlineAsm::ConstraintCode::m);
- // Find the uses to fold along with their defs, and the ties to keep. A
- // folded operand becomes MemOps.size() operands, moving every later one, and
- // MachineInstr can't move a tied operand. So untie everything (untying a use
- // also unties its def), re-add the operands from the first folded one on,
- // and re-tie the pairs that are still registers at their new positions.
- SmallVector<unsigned, 4> FoldOps(Ops);
+ // A folded operand becomes the operands of a memory reference, moving every
+ // later one, and MachineInstr can't move a tied operand. So untie everything
+ // (untying a use also unties its def), re-add the operands from the first
+ // folded one on, and re-tie the pairs that are still registers at their new
+ // positions.
SmallVector<std::pair<unsigned, unsigned>, 4> Ties;
for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
I != E; ++I) {
@@ -675,13 +673,14 @@ static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<unsigned> Ops,
continue;
MI.untieRegOperand(I);
- if (is_contained(Ops, DefIdx))
- FoldOps.push_back(I);
- else
+ if (!Fold[I])
Ties.emplace_back(DefIdx, I);
}
- unsigned First = *llvm::min_element(FoldOps);
+ unsigned First = InlineAsm::MIOp_FirstOperand;
+ while (!Fold[First])
+ ++First;
+
SmallVector<MachineOperand, 16> Tail(MI.operands_begin() + First,
MI.operands_end());
while (MI.getNumOperands() > First)
@@ -690,7 +689,7 @@ static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<unsigned> Ops,
SmallVector<unsigned, 16> NewIdx;
for (auto [I, MO] : enumerate(Tail)) {
NewIdx.push_back(MI.getNumOperands());
- if (!is_contained(FoldOps, First + I)) {
+ if (!Fold[First + I]) {
MI.addOperand(MO);
continue;
}
@@ -717,40 +716,52 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
ArrayRef<unsigned> Ops, int FI,
const TargetInstrInfo &TII) {
assert(MI.isInlineAsm() && "wrong opcode");
-
- // Sanity check that there are operands to fold.
if (Ops.empty())
return nullptr;
- // Every operand in Ops holds the same register (e.g. one value passed to two
- // "rm" operands), so they all fold to the same stack slot. The asm reads
- // the slot through a use, and through a def tied to a use, which need not
- // be the same virtual register when the allocator lowers ties itself. In
- // that case the caller must store the tied use's value into the slot before
- // the asm.
- bool Reads = false;
- bool Writes = false;
+ // A tied use is only ever folded along with its def.
+ Register Reg = MI.getOperand(Ops[0]).getReg();
for (unsigned Op : Ops) {
assert(Op && "should never be first operand");
- const MachineOperand &MO = MI.getOperand(Op);
- assert(MO.isReg() && "shouldn't be folding non-reg operands");
- assert(MO.getReg() == MI.getOperand(Ops[0]).getReg() &&
+ assert(MI.getOperand(Op).isReg() && MI.getOperand(Op).getReg() == Reg &&
"folded operands must hold the same register");
-
- // A tied use is only ever folded along with its def.
- if (!MI.mayFoldInlineAsmRegOp(Op))
+ if (MI.isRegTiedToDefOperand(Op))
return nullptr;
+ }
+
+ // The asm can't refer to both Reg and its stack slot, so every operand of
+ // Reg is folded (e.g. one value passed to two "rm" operands), including any
+ // the caller left out, such as an undef use.
+ if (!MI.mayFoldInlineAsmReg(Reg))
+ return nullptr;
+
+ // Find the operands to fold. A use tied to another register's def keeps its
+ // register. The use tied to one of Reg's defs is folded along with the def,
+ // and reads the def's slot. That use need not hold Reg when the allocator
+ // lowers ties itself; the caller must then store the tied use's value into
+ // the slot before the asm.
+ SmallVector<bool, 16> Fold(MI.getNumOperands());
+ bool Reads = false;
+ bool Writes = false;
+ for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
+ I != E; ++I) {
+ const MachineOperand &MO = MI.getOperand(I);
+ if (!MO.isReg())
+ continue;
+ unsigned DefIdx;
+ const MachineOperand &SlotMO =
+ MI.isRegTiedToDefOperand(I, &DefIdx) ? MI.getOperand(DefIdx) : MO;
+ if (SlotMO.getReg() != Reg)
+ continue;
+
+ Fold[I] = true;
Reads |= MO.readsReg();
- if (MO.isDef()) {
- Writes = true;
- Reads |=
- MO.isTied() && MI.getOperand(MI.findTiedOperandIdx(Op)).readsReg();
- }
+ Writes |= MO.isDef();
}
MachineInstr &NewMI = TII.duplicate(*MI.getParent(), MI.getIterator(), MI);
- foldInlineAsmMemOperands(NewMI, Ops, FI, TII);
+ foldInlineAsmMemOperands(NewMI, Fold, FI, TII);
// Update mayload/maystore metadata, and memoperands.
MachineOperand &ExtraMO = NewMI.getOperand(InlineAsm::MIOp_ExtraInfo);
diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
index eceb4d4d4bc37c..deec6136d87a53 100644
--- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
+++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
@@ -1,18 +1,17 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 6
-# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=greedy \
-# RUN: -verify-machineinstrs %s -o - | FileCheck %s
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=greedy %s -o - \
+# RUN: | FileCheck %s
# The greedy register allocator spills a value used by a foldable ("rm")
-# inline asm operand by folding the operand to the value's stack slot. The
-# clobbers below leave six GPRs, and the asm needs seven registers, so one
-# value is spilled. Folding replaces an operand with the five operands of a
-# memory reference, moving every operand after it, which must keep any tied
-# operands tied.
+# inline asm operand by folding the operand to the value's stack slot.
+# Folding replaces an operand with the five operands of a memory reference,
+# moving every operand after it, which must keep any tied operands tied.
#
# Each test is derived from the C function and inline asm statement in its
-# comment, where CLOBBERS is:
+# comment, where CLOBBERS leaves six GPRs, and ALL_GPRS none:
#
-# "rax", "rbx", "rbp", "r10", "r11", "r12", "r13", "r14", "r15"
+# CLOBBERS: "rax", "rbx", "rbp", "r10", "r11", "r12", "r13", "r14", "r15"
+# ALL_GPRS: CLOBBERS, "rcx", "rdx", "rsi", "rdi", "r8", "r9"
---
# One value passed to two "rm" operands: both operands fold to the same stack
@@ -374,3 +373,84 @@ body: |
INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %6, reguse:GR64 foldable, %6, reguse:GR64, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
RET 0
...
+---
+# One value passed to an "rm" and an "r" operand isn't folded, since the "r"
+# operand needs it in a register anyway, so the allocator must not count on
+# spilling it: %h, which only lives across the asm, is spilled instead.
+#
+# long rm_and_r_same_value(long a, long b, long c, long d, long e, long f,
+# long g) {
+# asm volatile("" : : : ALL_GPRS);
+# long h = f + 8;
+# asm volatile("" : : "rm"(g), "r"(g), "r"(a), "r"(b), "r"(c), "r"(d),
+# "r"(e) : CLOBBERS);
+# return h;
+# }
+name: rm_and_r_same_value
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 8, alignment: 16, isImmutable: true }
+body: |
+ bb.0:
+ liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-LABEL: name: rm_and_r_same_value
+ ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %stack.2)
+ ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.3)
+ ; CHECK-NEXT: MOV64mr %stack.4, 1, $noreg, 0, $noreg, $r8 :: (store (s64) into %stack.4)
+ ; CHECK-NEXT: MOV64mr %stack.6, 1, $noreg, 0, $noreg, $r9 :: (store (s64) into %stack.6)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.6, 1, $noreg, 0, $noreg :: (load (s64) from %stack.6)
+ ; CHECK-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r [[MOV64rm]], 1, $noreg, 8, $noreg
+ ; CHECK-NEXT: MOV64mr %stack.5, 1, $noreg, 0, $noreg, [[LEA64r]] :: (store (s64) into %stack.5)
+ ; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ ; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1)
+ ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2)
+ ; CHECK-NEXT: [[MOV64rm5:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
+ ; CHECK-NEXT: [[MOV64rm6:%[0-9]+]]:gr64 = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, [[MOV64rm1]], reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[MOV64rm5]], reguse:GR64, [[MOV64rm6]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: $rax = MOV64rm %stack.5, 1, $noreg, 0, $noreg :: (load (s64) from %stack.5)
+ ; CHECK-NEXT: RET 0, $rax
+ %0:gr64 = COPY $rdi
+ %1:gr64 = COPY $rsi
+ %2:gr64 = COPY $rdx
+ %3:gr64 = COPY $rcx
+ %4:gr64 = COPY $r8
+ %5:gr64 = COPY $r9
+ INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ %7:gr64 = LEA64r %5, 1, $noreg, 8, $noreg
+ %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %6, reguse:GR64, %6, reguse:GR64, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ $rax = COPY %7
+ RET 0, $rax
+...
+---
+# An undef use of a folded register is folded along with its other operands,
+# so that none is left referring to the register. The second asm clobbers
+# every GPR, so its operands must be folded. The third asm's undef use can't
+# be folded, so nothing is, and it gets a reload. (There is no C source:
+# instruction selection doesn't emit undef inline asm operands.)
+name: rm_undef_use
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rdi
+ ; CHECK-LABEL: name: rm_undef_use
+ ; CHECK: liveins: $rdi
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, [[MOV64rm]], reguse:GR64, undef [[MOV64rm]]
+ ; CHECK-NEXT: RET 0
+ %0:gr64 = COPY $rdi
+ INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %0, reguse:GR64 foldable, undef %0, reguse:GR64 foldable, %0, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %0, reguse:GR64, undef %0
+ RET 0
+...
>From 1bc587dfe5def971b7b52f3818a4b76b2560e55f Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Mon, 26 Oct 2026 00:18:20 -0700
Subject: [PATCH 6/9] [CodeGen] Fold an inline asm subregister operand at its
slot offset
A stack slot holds the whole register, and a folded operand referred to
the start of the slot, so a subregister was folded only if it was at
offset 0, and other subregisters got a reload into a register the asm
may not have. Coalescing can turn an i686 "rm" byte operand into
sub_8bit_hi of a GR32, which is at byte 1. Fold a subregister at its
offset in the slot, from getStackSlotRange(), so
getFrameIndexOperands() now takes an offset.
Also add a third asm to qm_tied_high_byte, which clobbers every GPR, so
that it reaches the folder. Without it, the value is reloaded into a
register that's free at the second asm.
Assisted-by: Claude Opus 5.5
---
llvm/include/llvm/CodeGen/MachineInstr.h | 4 +-
llvm/include/llvm/CodeGen/TargetInstrInfo.h | 5 +-
llvm/lib/CodeGen/MachineInstr.cpp | 39 ++++++-----
llvm/lib/CodeGen/TargetInstrInfo.cpp | 65 +++++++++++--------
llvm/lib/Target/X86/X86InstrInfo.cpp | 3 +-
llvm/lib/Target/X86/X86InstrInfo.h | 4 +-
.../CodeGen/X86/greedy-inline-asm-fold.mir | 36 +++++-----
7 files changed, 84 insertions(+), 72 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h
index dd689cf280f371..4185e8739d659b 100644
--- a/llvm/include/llvm/CodeGen/MachineInstr.h
+++ b/llvm/include/llvm/CodeGen/MachineInstr.h
@@ -1440,8 +1440,8 @@ class MachineInstr
}
/// Returns true if the register operand can be folded with a load or store
/// into a frame index. Does so by checking the InlineAsm::Flag immediate
- /// operand at OpId - 1, and that the register (or subregister) starts at the
- /// beginning of the stack slot.
+ /// operand at OpId - 1, and that the register (or subregister) has a place
+ /// in the register's stack slot.
LLVM_ABI bool mayFoldInlineAsmRegOp(unsigned OpId) const;
/// Returns true if \p Reg can be folded to its stack slot in this inline
diff --git a/llvm/include/llvm/CodeGen/TargetInstrInfo.h b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
index 97baf780275ee3..9bd88d716f4175 100644
--- a/llvm/include/llvm/CodeGen/TargetInstrInfo.h
+++ b/llvm/include/llvm/CodeGen/TargetInstrInfo.h
@@ -2418,13 +2418,14 @@ class LLVM_ABI TargetInstrInfo : public MCInstrInfo {
// Get the call frame size just before MI.
unsigned getCallFrameSizeAt(MachineInstr &MI) const;
- /// Fills in the necessary MachineOperands to refer to a frame index.
+ /// Fills in the necessary MachineOperands to refer to a frame index, plus
+ /// \p Offset bytes.
/// The best way to understand this is to print `asm(""::"m"(x));` after
/// finalize-isel. Example:
/// INLINEASM ... 262190 /* mem:m */, %stack.0.x.addr, 1, $noreg, 0, $noreg
/// we would add placeholders for: ^ ^ ^ ^
virtual void getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops,
- int FI) const {
+ int FI, int64_t Offset) const {
llvm_unreachable("unknown number of operands necessary");
}
diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp
index 23f5170ea66d20..ed7d88c9935b25 100644
--- a/llvm/lib/CodeGen/MachineInstr.cpp
+++ b/llvm/lib/CodeGen/MachineInstr.cpp
@@ -2784,37 +2784,36 @@ bool MachineInstr::mayFoldInlineAsmRegOp(unsigned OpId) const {
assert(OpId && "expected non-zero operand id");
assert(isInlineAsm() && "should only be used on inline asm");
- if (!getOperand(OpId).isReg())
- return false;
-
- if (isRegTiedToDefOperand(OpId))
+ const MachineOperand &MO = getOperand(OpId);
+ const MachineOperand &MD = getOperand(OpId - 1);
+ if (!MO.isReg() || !MD.isImm())
return false;
- const MachineOperand &MD = getOperand(OpId - 1);
- if (!MD.isImm())
+ InlineAsm::Flag F(MD.getImm());
+ if (!F.isRegUseKind() && !F.isRegDefKind() && !F.isRegDefEarlyClobberKind())
return false;
// A memory operand replaces exactly one register, so one register of a
// multi-register operand (e.g. an i128 in two GR64s) can't be folded.
- InlineAsm::Flag F(MD.getImm());
- if (F.getNumOperandRegisters() != 1)
+ if (!F.getRegMayBeFolded() || F.getNumOperandRegisters() != 1)
return false;
- // A memory operand refers to the start of the stack slot, which holds the
- // whole register. A subregister is found there only if it's at bit offset 0
- // on a little-endian target; e.g. x86's sub_8bit_hi is at byte 1. (The use
- // tied to a def, which is folded along with it, has the def's subregister.)
- if (unsigned SubReg = getOperand(OpId).getSubReg()) {
+ // A use tied to a def is folded along with the def, not on its own.
+ if (isRegTiedToDefOperand(OpId))
+ return false;
+
+ // The stack slot holds the whole register, so a subregister is folded at
+ // its offset in the slot, if it has one.
+ if (unsigned SubReg = MO.getSubReg()) {
const MachineFunction &MF = *getMF();
- if (!MF.getDataLayout().isLittleEndian() ||
- MF.getSubtarget().getRegisterInfo()->getSubRegIdxOffset(SubReg) != 0)
- return false;
+ unsigned Size, Offset;
+ return MO.getReg().isVirtual() &&
+ MF.getSubtarget().getInstrInfo()->getStackSlotRange(
+ MF.getRegInfo().getRegClass(MO.getReg()), SubReg, Size, Offset,
+ MF);
}
- if (F.isRegUseKind() || F.isRegDefKind() || F.isRegDefEarlyClobberKind())
- return F.getRegMayBeFolded();
-
- return false;
+ return true;
}
bool MachineInstr::mayFoldInlineAsmReg(Register Reg) const {
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 47bfa9cdc94610..5de5fac41b0005 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -650,16 +650,12 @@ static MachineInstr *foldPatchpoint(MachineFunction &MF, MachineInstr &MI,
return NewMI;
}
-/// Rewrite each register operand of the inline asm \p MI that's set in \p Fold
-/// (indexed by operand number) to refer to stack slot \p FI.
-static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<bool> Fold,
+/// Rewrite each register operand of the inline asm \p MI that has an entry in
+/// \p Offsets (indexed by operand number) to refer to stack slot \p FI at
+/// that byte offset.
+static void foldInlineAsmMemOperands(MachineInstr &MI,
+ ArrayRef<std::optional<unsigned>> Offsets,
int FI, const TargetInstrInfo &TII) {
- SmallVector<MachineOperand, 5> MemOps;
- TII.getFrameIndexOperands(MemOps, FI);
- assert(!MemOps.empty() && "getFrameIndexOperands didn't create any operands");
- InlineAsm::Flag MemFlag(InlineAsm::Kind::Mem, MemOps.size());
- MemFlag.setMemConstraint(InlineAsm::ConstraintCode::m);
-
// A folded operand becomes the operands of a memory reference, moving every
// later one, and MachineInstr can't move a tied operand. So untie everything
// (untying a use also unties its def), re-add the operands from the first
@@ -673,12 +669,12 @@ static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<bool> Fold,
continue;
MI.untieRegOperand(I);
- if (!Fold[I])
+ if (!Offsets[I])
Ties.emplace_back(DefIdx, I);
}
unsigned First = InlineAsm::MIOp_FirstOperand;
- while (!Fold[First])
+ while (!Offsets[First])
++First;
SmallVector<MachineOperand, 16> Tail(MI.operands_begin() + First,
@@ -689,7 +685,8 @@ static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<bool> Fold,
SmallVector<unsigned, 16> NewIdx;
for (auto [I, MO] : enumerate(Tail)) {
NewIdx.push_back(MI.getNumOperands());
- if (!Fold[First + I]) {
+ std::optional<unsigned> Offset = Offsets[First + I];
+ if (!Offset) {
MI.addOperand(MO);
continue;
}
@@ -699,6 +696,12 @@ static void foldInlineAsmMemOperands(MachineInstr &MI, ArrayRef<bool> Fold,
MachineOperand &FlagMO = MI.getOperand(MI.getNumOperands() - 1);
assert(InlineAsm::Flag(FlagMO.getImm()).getNumOperandRegisters() == 1 &&
"cannot fold one register of a multi-register operand");
+ SmallVector<MachineOperand, 5> MemOps;
+ TII.getFrameIndexOperands(MemOps, FI, *Offset);
+ assert(!MemOps.empty() &&
+ "getFrameIndexOperands didn't create any operands");
+ InlineAsm::Flag MemFlag(InlineAsm::Kind::Mem, MemOps.size());
+ MemFlag.setMemConstraint(InlineAsm::ConstraintCode::m);
FlagMO.setImm(MemFlag);
for (const MachineOperand &MemOp : MemOps)
MI.addOperand(MemOp);
@@ -735,12 +738,14 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
if (!MI.mayFoldInlineAsmReg(Reg))
return nullptr;
- // Find the operands to fold. A use tied to another register's def keeps its
- // register. The use tied to one of Reg's defs is folded along with the def,
- // and reads the def's slot. That use need not hold Reg when the allocator
- // lowers ties itself; the caller must then store the tied use's value into
- // the slot before the asm.
- SmallVector<bool, 16> Fold(MI.getNumOperands());
+ // Find the operands to fold and where each refers to in the slot, which
+ // holds the whole register: a subregister is at its offset in it. A use
+ // tied to another register's def keeps its register. The use tied to one of
+ // Reg's defs is folded along with the def, and reads the def's slot. That
+ // use need not hold Reg when the allocator lowers ties itself; the caller
+ // must then store the tied use's value into the slot before the asm.
+ MachineFunction &MF = *MI.getMF();
+ SmallVector<std::optional<unsigned>, 16> Offsets(MI.getNumOperands());
bool Reads = false;
bool Writes = false;
for (unsigned I = InlineAsm::MIOp_FirstOperand, E = MI.getNumOperands();
@@ -755,13 +760,21 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
if (SlotMO.getReg() != Reg)
continue;
- Fold[I] = true;
+ unsigned Offset = 0;
+ if (unsigned SubReg = SlotMO.getSubReg()) {
+ unsigned Size;
+ [[maybe_unused]] bool InSlot = TII.getStackSlotRange(
+ MF.getRegInfo().getRegClass(Reg), SubReg, Size, Offset, MF);
+ assert(InSlot && "foldable subregister isn't in the stack slot");
+ }
+
+ Offsets[I] = Offset;
Reads |= MO.readsReg();
Writes |= MO.isDef();
}
MachineInstr &NewMI = TII.duplicate(*MI.getParent(), MI.getIterator(), MI);
- foldInlineAsmMemOperands(NewMI, Fold, FI, TII);
+ foldInlineAsmMemOperands(NewMI, Offsets, FI, TII);
// Update mayload/maystore metadata, and memoperands.
MachineOperand &ExtraMO = NewMI.getOperand(InlineAsm::MIOp_ExtraInfo);
@@ -774,12 +787,12 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
ExtraMO.setImm(ExtraMO.getImm() | InlineAsm::Extra_MayStore);
Flags |= MachineMemOperand::MOStore;
}
- MachineFunction *MF = NewMI.getMF();
- const MachineFrameInfo &MFI = MF->getFrameInfo();
- MachineMemOperand *MMO = MF->getMachineMemOperand(
- MachinePointerInfo::getFixedStack(*MF, FI), Flags, MFI.getObjectSize(FI),
- MFI.getObjectAlign(FI));
- NewMI.addMemOperand(*MF, MMO);
+
+ const MachineFrameInfo &MFI = MF.getFrameInfo();
+ MachineMemOperand *MMO =
+ MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI), Flags,
+ MFI.getObjectSize(FI), MFI.getObjectAlign(FI));
+ NewMI.addMemOperand(MF, MMO);
return &NewMI;
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 251fb43e56000b..a38d1b36576205 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -11031,10 +11031,11 @@ void X86InstrInfo::genAlternativeCodeSequence(
// See also: X86DAGToDAGISel::SelectInlineAsmMemoryOperand().
void X86InstrInfo::getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops,
- int FI) const {
+ int FI, int64_t Offset) const {
X86AddressMode M;
M.BaseType = X86AddressMode::FrameIndexBase;
M.Base.FrameIndex = FI;
+ M.Disp = Offset;
M.getFullAddress(Ops);
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index b8c1aa6b296905..cb2706ce47d59d 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -703,8 +703,8 @@ class X86InstrInfo final : public X86GenInstrInfo {
return false;
}
- void getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops,
- int FI) const override;
+ void getFrameIndexOperands(SmallVectorImpl<MachineOperand> &Ops, int FI,
+ int64_t Offset) const override;
private:
/// This is a helper for convertToThreeAddress for 8 and 16-bit instructions.
diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
index deec6136d87a53..311b969a71a3c9 100644
--- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
+++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
@@ -224,13 +224,12 @@ body: |
RET 0
...
---
-# A stack slot holds the whole register, so a subregister operand can be
-# folded only if the subregister starts at the beginning of the slot. Each of
-# these tests spills a value across an asm that clobbers every GPR, as above,
-# and passes its low byte, its second byte, or both to a "Qm" operand of a
-# second asm.
+# A stack slot holds the whole register, so a subregister operand is folded
+# at the subregister's offset in the slot. Each of these tests spills a value
+# across an asm that clobbers every GPR, as above, and passes its low byte,
+# its second byte, or both to a "Qm" operand of a second asm.
#
-# The low byte (sub_8bit) is at the start of the slot, so it's folded.
+# The low byte (sub_8bit) is at offset 0.
#
# void qm_low_byte(unsigned long a) {
# asm volatile("" : : : ALL_GPRS);
@@ -254,8 +253,7 @@ body: |
RET 0
...
---
-# The second byte (sub_8bit_hi) is at byte 1 of the slot, so it gets a reload
-# instead of a fold to the start of the slot.
+# The second byte (sub_8bit_hi) is at offset 1.
#
# void qm_high_byte(unsigned long a) {
# asm volatile("" : : : ALL_GPRS);
@@ -271,8 +269,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
- ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
- ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_H foldable, [[MOV64rm]].sub_8bit_hi
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 1, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: RET 0
%0:gr64_abcd = COPY $rdi
INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
@@ -280,8 +277,7 @@ body: |
RET 0
...
---
-# Both bytes are subregisters of one value. Folding both operands would read
-# byte 0 for each, so neither is folded.
+# Both bytes are subregisters of one value, each folded at its own offset.
#
# void qm_both_bytes(unsigned long a) {
# asm volatile("" : : : ALL_GPRS);
@@ -298,8 +294,7 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
- ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
- ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, reguse:GR8_ABCD_L foldable, [[MOV64rm]].sub_8bit, reguse:GR8_ABCD_H foldable, [[MOV64rm]].sub_8bit_hi
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 1, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: RET 0
%0:gr64_abcd = COPY $rdi
INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
@@ -307,12 +302,14 @@ body: |
RET 0
...
---
-# A def is folded along with the use tied to it, so neither is folded when the
-# tied use is the second byte.
+# A def is folded along with the use tied to it, both at the offset of the
+# def's subregister. The third asm keeps the value in its slot after the
+# second.
#
# unsigned long qm_tied_high_byte(unsigned long a) {
# asm volatile("" : : : ALL_GPRS);
# asm volatile("" : "+Qm"(((unsigned char *)&a)[1]));
+# asm volatile("" : : : ALL_GPRS);
# return a;
# }
name: qm_tied_high_byte
@@ -325,13 +322,14 @@ body: |
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
- ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64_abcd = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
- ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, regdef:GR8_ABCD_H foldable, def [[MOV64rm]].sub_8bit_hi, reguse tiedto:$0, [[MOV64rm]].sub_8bit_hi(tied-def 3)
- ; CHECK-NEXT: $rax = COPY [[MOV64rm]]
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 1, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 1, $noreg :: (load store (s64) on %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ ; CHECK-NEXT: $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: RET 0, $rax
%0:gr64_abcd = COPY $rdi
INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
INLINEASM &"", sideeffect attdialect, regdef:GR8_ABCD_H foldable, def %0.sub_8bit_hi, reguse tiedto:$0, %0.sub_8bit_hi(tied-def 3)
+ INLINEASM &"", sideeffect attdialect, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rcx, clobber, implicit-def dead early-clobber $rdx, clobber, implicit-def dead early-clobber $rsi, clobber, implicit-def dead early-clobber $rdi, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r8, clobber, implicit-def dead early-clobber $r9, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
$rax = COPY %0
RET 0, $rax
...
>From 5f2e80314563582d1cb37619e2e0bf500e39d7e3 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Mon, 26 Oct 2026 00:19:57 -0700
Subject: [PATCH 7/9] [CodeGen] Prefer spilling a register an inline asm can
fold
InlineSpiller rematerialized a value passed to a foldable inline asm
operand (e.g. a constant) into a register right before the asm, which
the asm may not have had, since registers running out is why the value
was spilled. It's now spilled and folded instead.
An asm that can fold a register also no longer adds to the register's
spill weight, since it needs no reload or spill for it. The allocator
then spills that register before the asm's other operands, which need
registers, which saves spills in two_tied_rm and tied_rm_undef.
Assisted-by: Claude Opus 5.5
---
llvm/lib/CodeGen/CalcSpillWeights.cpp | 6 +-
llvm/lib/CodeGen/InlineSpiller.cpp | 6 ++
.../CodeGen/X86/greedy-inline-asm-fold.mir | 70 +++++++++++++------
3 files changed, 61 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/CodeGen/CalcSpillWeights.cpp b/llvm/lib/CodeGen/CalcSpillWeights.cpp
index 26d265c2a986e8..81bf22243b06ec 100644
--- a/llvm/lib/CodeGen/CalcSpillWeights.cpp
+++ b/llvm/lib/CodeGen/CalcSpillWeights.cpp
@@ -319,7 +319,11 @@ float VirtRegAuxInfo::weightCalcHelper(LiveInterval &LI) {
// Force Weight onto the stack so that x86 doesn't add hidden precision.
stack_float_t Weight = 1.0f;
- if (IsSpillable) {
+ // An inline asm that can access the register in its stack slot needs no
+ // reload or spill for it, so it doesn't add to the spill weight. That makes
+ // the register a better choice to spill than the asm's other operands,
+ // which need registers.
+ if (IsSpillable && !canMemFoldInlineAsm(*MI, LI.reg())) {
// Get loop info for mi.
if (MI->getParent() != MBB) {
MBB = MI->getParent();
diff --git a/llvm/lib/CodeGen/InlineSpiller.cpp b/llvm/lib/CodeGen/InlineSpiller.cpp
index bdcf13266ae5a3..a313852443368c 100644
--- a/llvm/lib/CodeGen/InlineSpiller.cpp
+++ b/llvm/lib/CodeGen/InlineSpiller.cpp
@@ -621,6 +621,12 @@ void InlineSpiller::markValueUsed(LiveInterval *LI, VNInfo *VNI) {
bool llvm::canGuaranteeAssignmentAfterRemat(Register VReg,
const MachineInstr &MI) {
+ // An inline asm that can access VReg in its stack slot (an "rm" operand)
+ // falls back on that when registers run out, which is why VReg is being
+ // spilled. A register rematerialized for it may not be available.
+ if (MI.isInlineAsm() && MI.mayFoldInlineAsmReg(VReg))
+ return false;
+
if (!RestrictStatepointRemat)
return true;
// Here's a quick explanation of the problem we're trying to handle here:
diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
index 311b969a71a3c9..1f870984228c17 100644
--- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
+++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
@@ -117,20 +117,19 @@ body: |
; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: MOV64mr %stack.4, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.4)
+ ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.3)
; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %stack.0)
; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.1)
; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $r8 :: (store (s64) into %stack.2)
- ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $r9 :: (store (s64) into %stack.3)
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $r9
; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = lr-split COPY [[COPY]]
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = lr-split COPY [[COPY]]
; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1)
; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2)
- ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
- ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def [[COPY1]], mem:m ops:5, %stack.4, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[MOV64rm]], reguse tiedto:$0, [[COPY1]](tied-def 3), mem:m ops:5, %stack.4, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.4)
- ; CHECK-NEXT: $rax = COPY [[COPY1]]
- ; CHECK-NEXT: $rdx = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, regdef:GR64 foldable, def [[COPY2]], mem:m ops:5, %stack.3, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[COPY1]], reguse:GR64, [[MOV64rm]], reguse tiedto:$0, [[COPY2]](tied-def 3), mem:m ops:5, %stack.3, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load store (s64) on %stack.3)
+ ; CHECK-NEXT: $rax = COPY [[COPY2]]
+ ; CHECK-NEXT: $rdx = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
; CHECK-NEXT: RET 0, $rax, $rdx
%0:gr64 = COPY $rdi
%1:gr64 = COPY $rsi
@@ -167,19 +166,14 @@ body: |
; CHECK-LABEL: name: tied_rm_undef
; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: MOV64mr %stack.0, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.0)
- ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.1)
- ; CHECK-NEXT: MOV64mr %stack.2, 1, $noreg, 0, $noreg, $rdx :: (store (s64) into %stack.2)
- ; CHECK-NEXT: MOV64mr %stack.3, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.3)
- ; CHECK-NEXT: MOV64mr %stack.4, 1, $noreg, 0, $noreg, $r8 :: (store (s64) into %stack.4)
- ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $r9
- ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
- ; CHECK-NEXT: [[MOV64rm1:%[0-9]+]]:gr64 = MOV64rm %stack.1, 1, $noreg, 0, $noreg :: (load (s64) from %stack.1)
- ; CHECK-NEXT: [[MOV64rm2:%[0-9]+]]:gr64 = MOV64rm %stack.2, 1, $noreg, 0, $noreg :: (load (s64) from %stack.2)
- ; CHECK-NEXT: [[MOV64rm3:%[0-9]+]]:gr64 = MOV64rm %stack.3, 1, $noreg, 0, $noreg :: (load (s64) from %stack.3)
- ; CHECK-NEXT: [[MOV64rm4:%[0-9]+]]:gr64 = MOV64rm %stack.4, 1, $noreg, 0, $noreg :: (load (s64) from %stack.4)
- ; CHECK-NEXT: INLINEASM &"", sideeffect maystore attdialect, mem:m ops:5, %stack.5, 1, $noreg, 0, $noreg, reguse:GR64, [[MOV64rm]], reguse:GR64, [[MOV64rm1]], reguse:GR64, [[MOV64rm2]], reguse:GR64, [[MOV64rm3]], reguse:GR64, [[MOV64rm4]], reguse:GR64, [[COPY]], mem:m ops:5, %stack.5, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.5)
- ; CHECK-NEXT: $rax = MOV64rm %stack.5, 1, $noreg, 0, $noreg :: (load (s64) from %stack.5)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rdx
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $rcx
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r8
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY $r9
+ ; CHECK-NEXT: INLINEASM &"", sideeffect maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[COPY5]], mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: $rax = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
; CHECK-NEXT: RET 0, $rax
%1:gr64 = COPY $rdi
%2:gr64 = COPY $rsi
@@ -452,3 +446,39 @@ body: |
INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %0, reguse:GR64, undef %0
RET 0
...
+---
+# A constant passed to an "rm" operand is spilled and folded rather than
+# rematerialized into a register, which would need a register that the asm
+# doesn't leave.
+#
+# void rm_constant(long a, long b, long c, long d, long e, long f) {
+# asm volatile("" : : "rm"(42L), "r"(a), "r"(b), "r"(c), "r"(d), "r"(e),
+# "r"(f) : CLOBBERS);
+# }
+name: rm_constant
+tracksRegLiveness: true
+body: |
+ bb.0:
+ liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-LABEL: name: rm_constant
+ ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rdx
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $rcx
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r8
+ ; CHECK-NEXT: [[COPY5:%[0-9]+]]:gr64 = COPY $r9
+ ; CHECK-NEXT: MOV64mi32 %stack.0, 1, $noreg, 0, $noreg, 42 :: (store (s64) into %stack.0)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[COPY5]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (load (s64) from %stack.0)
+ ; CHECK-NEXT: RET 0
+ %0:gr64 = COPY $rdi
+ %1:gr64 = COPY $rsi
+ %2:gr64 = COPY $rdx
+ %3:gr64 = COPY $rcx
+ %4:gr64 = COPY $r8
+ %5:gr64 = COPY $r9
+ %6:gr64 = MOV64ri32 42
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %6, reguse:GR64, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ RET 0
+...
>From c2f5cf3cfaa6dd4d50df3bcdbfdfb6c1755a5729 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Mon, 26 Oct 2026 00:20:40 -0700
Subject: [PATCH 8/9] [CodeGen] Keep an inline asm's memory accesses described
after a fold
The memoperands of an instruction describe all of its memory accesses.
Folding gave an inline asm a single memoperand, for the stack slot,
which said that it accessed no other memory even when it had an "m"
operand or a "memory" clobber. Without memoperands, the asm was taken to
access any memory, so it now gets one for any memory as well.
Assisted-by: Claude Opus 5.5
---
llvm/lib/CodeGen/TargetInstrInfo.cpp | 15 +++++++
.../CodeGen/X86/greedy-inline-asm-fold.mir | 41 +++++++++++++++++++
2 files changed, 56 insertions(+)
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 5de5fac41b0005..81cd2cfc936932 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -788,6 +788,21 @@ static MachineInstr *foldInlineAsmMemOperand(MachineInstr &MI,
Flags |= MachineMemOperand::MOStore;
}
+ // The memoperands of an instruction describe all of its memory accesses. An
+ // asm without any may still access memory, e.g. through an "m" operand or a
+ // "memory" clobber, so give it one for any memory as well as the slot's.
+ if (MI.memoperands_empty() && (MI.mayLoad() || MI.mayStore())) {
+ MachineMemOperand::Flags OtherFlags = MachineMemOperand::MONone;
+ if (MI.mayLoad())
+ OtherFlags |= MachineMemOperand::MOLoad;
+ if (MI.mayStore())
+ OtherFlags |= MachineMemOperand::MOStore;
+ NewMI.addMemOperand(
+ MF, MF.getMachineMemOperand(MachinePointerInfo(), OtherFlags,
+ LocationSize::beforeOrAfterPointer(),
+ Align(1)));
+ }
+
const MachineFrameInfo &MFI = MF.getFrameInfo();
MachineMemOperand *MMO =
MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI), Flags,
diff --git a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
index 1f870984228c17..5b9c791d65e7ae 100644
--- a/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
+++ b/llvm/test/CodeGen/X86/greedy-inline-asm-fold.mir
@@ -447,6 +447,47 @@ body: |
RET 0
...
---
+# The memoperands of an instruction describe all of its memory accesses, so
+# the stack slot's isn't added to an asm that accesses other memory without
+# memoperands, here through an "=m" output.
+#
+# void rm_and_m(long a, long b, long c, long d, long e, long f, long g) {
+# long x;
+# asm volatile("" : "=m"(x) : "rm"(a), "r"(b), "r"(c), "r"(d), "r"(e),
+# "r"(f), "r"(g) : CLOBBERS);
+# }
+name: rm_and_m
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 8, alignment: 16, isImmutable: true }
+stack:
+ - { id: 0, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-LABEL: name: rm_and_m
+ ; CHECK: liveins: $rdi, $rsi, $rdx, $rcx, $r8, $r9
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rdi :: (store (s64) into %stack.1)
+ ; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rsi
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr64 = COPY $rdx
+ ; CHECK-NEXT: [[COPY2:%[0-9]+]]:gr64 = COPY $rcx
+ ; CHECK-NEXT: [[COPY3:%[0-9]+]]:gr64 = COPY $r8
+ ; CHECK-NEXT: [[COPY4:%[0-9]+]]:gr64 = COPY $r9
+ ; CHECK-NEXT: [[MOV64rm:%[0-9]+]]:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ ; CHECK-NEXT: INLINEASM &"", sideeffect mayload maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.1, 1, $noreg, 0, $noreg, reguse:GR64, [[COPY]], reguse:GR64, [[COPY1]], reguse:GR64, [[COPY2]], reguse:GR64, [[COPY3]], reguse:GR64, [[COPY4]], reguse:GR64, [[MOV64rm]], clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15 :: (store unknown-size, align 1), (load (s64) from %stack.1)
+ ; CHECK-NEXT: RET 0
+ %0:gr64 = COPY $rdi
+ %1:gr64 = COPY $rsi
+ %2:gr64 = COPY $rdx
+ %3:gr64 = COPY $rcx
+ %4:gr64 = COPY $r8
+ %5:gr64 = COPY $r9
+ %6:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ INLINEASM &"", sideeffect maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64 foldable, %0, reguse:GR64, %1, reguse:GR64, %2, reguse:GR64, %3, reguse:GR64, %4, reguse:GR64, %5, reguse:GR64, %6, clobber, implicit-def dead early-clobber $rax, clobber, implicit-def dead early-clobber $rbx, clobber, implicit-def dead early-clobber $rbp, clobber, implicit-def dead early-clobber $r10, clobber, implicit-def dead early-clobber $r11, clobber, implicit-def dead early-clobber $r12, clobber, implicit-def dead early-clobber $r13, clobber, implicit-def dead early-clobber $r14, clobber, implicit-def dead early-clobber $r15
+ RET 0
+...
+---
# A constant passed to an "rm" operand is spilled and folded rather than
# rematerialized into a register, which would need a register that the asm
# doesn't leave.
>From 2796e585d3eb7a9cefa6a4f5db5c3121144c5fb3 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Mon, 26 Oct 2026 00:21:36 -0700
Subject: [PATCH 9/9] [CodeGen] Don't fold a load into an inline asm that may
overwrite it
The peephole optimizer folds a load from a stack slot into a foldable
inline asm operand, which then reads the slot itself. An asm that writes
the slot first (through an early-clobber "=&m" output) then read what it
wrote, so a load isn't folded into an asm that stores, unless the slot
is invariant. Nor is a load that doesn't fill the register, e.g. a
32-bit load into an XMM register, since the asm would read the whole
register's worth of memory.
Assisted-by: Claude Opus 5.5
---
llvm/lib/CodeGen/TargetInstrInfo.cpp | 14 ++-
.../CodeGen/X86/peephole-inline-asm-fold.mir | 100 ++++++++++++++++++
2 files changed, 113 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir
diff --git a/llvm/lib/CodeGen/TargetInstrInfo.cpp b/llvm/lib/CodeGen/TargetInstrInfo.cpp
index 81cd2cfc936932..81b61333fce48c 100644
--- a/llvm/lib/CodeGen/TargetInstrInfo.cpp
+++ b/llvm/lib/CodeGen/TargetInstrInfo.cpp
@@ -929,6 +929,7 @@ TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops,
// Ask the target to do the actual folding.
MachineInstr *NewMI = nullptr;
int FrameIndex = 0;
+ TypeSize MemBytes = TypeSize::getZero();
if ((MI.getOpcode() == TargetOpcode::STACKMAP ||
MI.getOpcode() == TargetOpcode::PATCHPOINT ||
@@ -938,7 +939,18 @@ TargetInstrInfo::foldMemoryOperand(MachineInstr &MI, ArrayRef<unsigned> Ops,
NewMI = foldPatchpoint(MF, MI, Ops, FrameIndex, *this);
if (NewMI)
NewMI = &*MBB.insert(MI, NewMI);
- } else if (MI.isInlineAsm() && isLoadFromStackSlot(LoadMI, FrameIndex)) {
+ } else if (MI.isInlineAsm() &&
+ isLoadFromStackSlot(LoadMI, FrameIndex, MemBytes)) {
+ // The asm reads the stack slot in place of the loaded register, so the
+ // load must fill the register, and the slot must still hold the value when
+ // the asm reads it. An asm that stores might write the slot first (e.g.
+ // through an early-clobber "=&m" output), unless it's invariant.
+ if (Ops.empty() ||
+ MemBytes * 8 != TRI.getRegSizeInBits(MI.getOperand(Ops[0]).getReg(),
+ MF.getRegInfo()) ||
+ (MI.mayStore() && !LoadMI.isDereferenceableInvariantLoad()))
+ return nullptr;
+
return foldInlineAsmMemOperand(MI, Ops, FrameIndex, *this);
} else {
// Ask the target to do the actual folding.
diff --git a/llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir b/llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir
new file mode 100644
index 00000000000000..6e2d22c75ad376
--- /dev/null
+++ b/llvm/test/CodeGen/X86/peephole-inline-asm-fold.mir
@@ -0,0 +1,100 @@
+# RUN: llc -mtriple=x86_64-unknown-linux-gnu -run-pass=peephole-opt %s -o - \
+# RUN: | FileCheck %s
+
+# The peephole optimizer folds a load from a stack slot into a foldable ("rm")
+# inline asm operand that reads the loaded value, so the asm reads the slot
+# itself.
+
+--- |
+ define void @fold_load() { ret void }
+ define void @asm_may_store() { %x = alloca i64, align 8
+ ret void }
+ define void @invariant_load() { ret void }
+ define void @narrow_load() { ret void }
+ define void @undef_uses() { ret void }
+...
+---
+# CHECK-LABEL: name: fold_load
+# CHECK: INLINEASM &"", sideeffect mayload attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+name: fold_load
+tracksRegLiveness: true
+stack:
+ - { id: 0, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ %0:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, %0, reguse:GR64 foldable, %0
+ RET 0
+...
+---
+# The asm writes x through an early-clobber output before it reads v, so v
+# can't be read from x.
+#
+# long x;
+# long v = x;
+# asm volatile("movq $0, %0; movq %1, %%rax" : "=&m"(x) : "rm"(v) : "rax");
+#
+# CHECK-LABEL: name: asm_may_store
+# CHECK: %0:gr64 = MOV64rm %stack.0.x
+# CHECK-NEXT: INLINEASM {{.*}}, reguse:GR64 foldable, %0,
+name: asm_may_store
+tracksRegLiveness: true
+stack:
+ - { id: 0, name: x, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ %0:gr64 = MOV64rm %stack.0.x, 1, $noreg, 0, $noreg :: (dereferenceable load (s64) from %ir.x)
+ INLINEASM &"movq $$0, $0\0A\09movq $1, %rax", sideeffect mayload maystore attdialect, mem:m ops:5, %stack.0.x, 1, $noreg, 0, $noreg, reguse:GR64 foldable, %0, clobber, implicit-def early-clobber $rax
+ RET 0
+...
+---
+# An asm that stores can't write an invariant slot, such as an incoming
+# argument's.
+#
+# CHECK-LABEL: name: invariant_load
+# CHECK: INLINEASM &"", sideeffect mayload maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, mem:m ops:5, %fixed-stack.0, 1, $noreg, 0, $noreg :: (store unknown-size, align 1), (load (s64) from %fixed-stack.0, align 16)
+name: invariant_load
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 8, alignment: 16, isImmutable: true }
+stack:
+ - { id: 0, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ %0:gr64 = MOV64rm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %fixed-stack.0, align 16)
+ INLINEASM &"", sideeffect maystore attdialect, mem:m ops:5, %stack.0, 1, $noreg, 0, $noreg, reguse:GR64 foldable, %0
+ RET 0
+...
+---
+# A load that fills only part of the register (here the low 4 bytes of an
+# XMM register, with the rest zeroed) isn't folded, since the asm would read
+# the whole register's width from the slot.
+#
+# CHECK-LABEL: name: narrow_load
+# CHECK: %0:vr128 = MOVSSrm %fixed-stack.0
+# CHECK-NEXT: INLINEASM &"movaps $0, %xmm1", sideeffect attdialect, reguse:VR128 foldable, %0
+name: narrow_load
+tracksRegLiveness: true
+fixedStack:
+ - { id: 0, size: 4, alignment: 16, isImmutable: true }
+body: |
+ bb.0:
+ %0:vr128 = MOVSSrm %fixed-stack.0, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.0, align 16)
+ INLINEASM &"movaps $0, %xmm1", sideeffect attdialect, reguse:VR128 foldable, %0
+ RET 0
+...
+---
+# An asm that doesn't read the loaded value has nothing to fold.
+#
+# CHECK-LABEL: name: undef_uses
+# CHECK: INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, undef %0, reguse:GR64 foldable, undef %0
+name: undef_uses
+tracksRegLiveness: true
+stack:
+ - { id: 0, size: 8, alignment: 8 }
+body: |
+ bb.0:
+ %0:gr64 = MOV64rm %stack.0, 1, $noreg, 0, $noreg :: (load (s64) from %stack.0)
+ INLINEASM &"", sideeffect attdialect, reguse:GR64 foldable, undef %0, reguse:GR64 foldable, undef %0
+ RET 0
+...
More information about the llvm-branch-commits
mailing list