[llvm] [CodeGen][ISel] Fix inline assembly crash during instruction selection with direct memory output constraint (PR #209439)
Chibuoyim Ogbonna via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 14 06:00:36 PDT 2026
https://github.com/bruteforceboy updated https://github.com/llvm/llvm-project/pull/209439
>From 3e9770841315eb184b4e5bf65a2322d5ea7d5857 Mon Sep 17 00:00:00 2001
From: workwilson <ogbonnachibuoyim12 at gmail.com>
Date: Tue, 14 Jul 2026 18:49:31 +0800
Subject: [PATCH 1/3] [CodeGen][ISel] Fix inline assembly crash during
instruction selection with direct memory output constraint
---
.../CodeGen/GlobalISel/InlineAsmLowering.cpp | 13 +++++
.../SelectionDAG/SelectionDAGBuilder.cpp | 24 +++++++---
.../CodeGen/SelectionDAG/TargetLowering.cpp | 15 ++++++
.../inline-asm-direct-mem-output-error.ll | 14 ++++++
.../AArch64/inline-asm-direct-mem-output.ll | 48 +++++++++++++++++++
.../X86/inline-asm-direct-mem-output-error.ll | 13 +++++
.../X86/inline-asm-direct-mem-output.ll | 36 ++++++++++++++
7 files changed, 157 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
create mode 100644 llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output.ll
create mode 100644 llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
create mode 100644 llvm/test/CodeGen/X86/inline-asm-direct-mem-output.ll
diff --git a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
index b5bbcc193b6b7..9c939e24db1dc 100644
--- a/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/InlineAsmLowering.cpp
@@ -333,6 +333,19 @@ bool InlineAsmLowering::lowerInlineAsm(
switch (OpInfo.Type) {
case InlineAsm::isOutput:
if (OpInfo.ConstraintType == TargetLowering::C_Memory) {
+ // A memory output writes through an address passed to the asm, so it
+ // only has somewhere to write to if it is indirect (e.g. "=*m"). A
+ // direct memory output has no operand to name the memory, and no
+ // register alternative to fall back on.
+ if (!OpInfo.isIndirect) {
+ emitInlineAsmError(MIRBuilder, Call,
+ "memory output constraint '" +
+ Twine(OpInfo.ConstraintCode) +
+ "' must be indirect",
+ GetOrCreateVRegs(Call));
+ return true;
+ }
+
const InlineAsm::ConstraintCode ConstraintID =
TLI->getInlineAsmMemConstraint(OpInfo.ConstraintCode);
assert(ConstraintID != InlineAsm::ConstraintCode::Unknown &&
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 98c91e65b4752..94cb06cb52f97 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -10386,8 +10386,9 @@ constructOperandInfo(ConstraintDecisionInfo &Info,
return false;
}
-/// Compute which constraint option to use for each operand.
-static void
+/// Compute which constraint option to use for each operand. Returns true if an
+/// error was encountered, in which case Info.ErrorMsg describes it.
+static bool
computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
TargetLowering::AsmOperandInfoVector &TargetConstraints,
SelectionDAGBuilder &Builder, const TargetLowering &TLI,
@@ -10449,9 +10450,16 @@ computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
// need to provide an address for the memory input.
if (OpInfo.ConstraintType == TargetLowering::C_Memory &&
!OpInfo.isIndirect) {
- assert((OpInfo.isMultipleAlternative ||
- (OpInfo.Type == InlineAsm::isInput)) &&
- "Can only indirectify direct input operands!");
+ // Only an input can be indirectified: it has a value whose address we can
+ // take. A direct output becomes the result of the asm and has no operand
+ // naming memory to write through. Constraint selection already avoids
+ // memory for a direct output when a register alternative exists (e.g.
+ // "=rm"), so getting here means memory was the only choice (e.g. "=m").
+ if (OpInfo.Type != InlineAsm::isInput) {
+ Info.ErrorMsg << "memory output constraint '" << OpInfo.ConstraintCode
+ << "' must be indirect";
+ return true;
+ }
// Memory operands really want the address of the value.
Info.Chain = getAddressForMemoryInput(Info.Chain, Builder.getCurSDLoc(),
@@ -10464,6 +10472,8 @@ computeConstraintToUse(ConstraintDecisionInfo &Info, const CallBase &Call,
OpInfo.isIndirect = true;
}
}
+
+ return false;
}
/// Prepare DAG-level operands. As part of this, assign virtual and physical
@@ -10757,7 +10767,9 @@ determineConstraints(ConstraintDecisionInfo &Info,
Info.Chain = Builder.lowerStartEH(Info.Chain, EHPadBB, Info.BeginLabel);
// Second pass: Compute which constraint option to use.
- computeConstraintToUse(Info, Call, TargetConstraints, Builder, TLI, TM, DAG);
+ if (computeConstraintToUse(Info, Call, TargetConstraints, Builder, TLI, TM,
+ DAG))
+ return true;
// AsmNodeOperands - The operands for the ISD::INLINEASM node.
Info.AsmNodeOperands.push_back(SDValue()); // reserve space for input chain
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index bca34c5c347ee..5cada45bd7d2f 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -6416,6 +6416,21 @@ TargetLowering::ConstraintGroup TargetLowering::getConstraintPreferences(
Ret.emplace_back(Code, CType);
}
+ // A direct output becomes the result of the asm rather than being written
+ // through an address supplied by the caller, so it has to live in a
+ // register. Prefer the other alternatives, if any (e.g. "r" in "=rm").
+ // If memory is the only option (e.g. "=m"), keep it so that lowering can
+ // diagnose the constraint; memory outputs must be spelled indirectly
+ // (e.g. "=*m").
+ if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
+ auto IsMemOrAddr = [](const ConstraintPair &P) {
+ return P.second == TargetLowering::C_Memory ||
+ P.second == TargetLowering::C_Address;
+ };
+ if (!llvm::all_of(Ret, IsMemOrAddr))
+ llvm::erase_if(Ret, IsMemOrAddr);
+ }
+
llvm::stable_sort(Ret, [](ConstraintPair a, ConstraintPair b) {
return getConstraintPiority(a.second) > getConstraintPiority(b.second);
});
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
new file mode 100644
index 0000000000000..4f292d2d4105c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
@@ -0,0 +1,14 @@
+; RUN: not llc -mtriple=aarch64-unknown-linux-gnu < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=aarch64-unknown-linux-gnu -global-isel < %s 2>&1 | FileCheck %s
+
+; CHECK: error: memory output constraint 'm' must be indirect
+define i64 @direct_m_output() {
+ %v = call i64 asm "", "=m"()
+ ret i64 %v
+}
+
+; CHECK: error: memory output constraint '{{[mo]}}' must be indirect
+define i64 @direct_mo_output() {
+ %v = call i64 asm "", "=mo"()
+ ret i64 %v
+}
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output.ll b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output.ll
new file mode 100644
index 0000000000000..91baf51047b2e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output.ll
@@ -0,0 +1,48 @@
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu -global-isel < %s | FileCheck %s
+
+define i64 @direct_rm_output() {
+; CHECK-LABEL: direct_rm_output:
+; CHECK: // %bb.0:
+; CHECK-NEXT: //APP
+; CHECK-NEXT: mov x0, #42 // =0x2a
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: ret
+ %v = call i64 asm "mov $0, #42", "=rm"()
+ ret i64 %v
+}
+
+define i64 @direct_rm_output_used(i64 %x) {
+; CHECK-LABEL: direct_rm_output_used:
+; CHECK: // %bb.0:
+; CHECK-NEXT: //APP
+; CHECK-NEXT: mov x8, #42 // =0x2a
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+ %v = call i64 asm "mov $0, #42", "=rm"()
+ %s = add i64 %v, %x
+ ret i64 %s
+}
+
+define i64 @direct_rm_output_tied(i64 %x) {
+; CHECK-LABEL: direct_rm_output_tied:
+; CHECK: // %bb.0:
+; CHECK-NEXT: //APP
+; CHECK-NEXT: add x0, x0, #1
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: ret
+ %v = call i64 asm "add $0, $0, #1", "=rm,0"(i64 %x)
+ ret i64 %v
+}
+
+define void @indirect_m_output(ptr %p) {
+; CHECK-LABEL: indirect_m_output:
+; CHECK: // %bb.0:
+; CHECK-NEXT: //APP
+; CHECK-NEXT: str xzr, [x0]
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: ret
+ call void asm "str xzr, $0", "=*m"(ptr elementtype(i64) %p)
+ ret void
+}
diff --git a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
new file mode 100644
index 0000000000000..121fa8729e4c8
--- /dev/null
+++ b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
@@ -0,0 +1,13 @@
+; RUN: not llc -mtriple=x86_64-unknown-linux-gnu < %s 2>&1 | FileCheck %s
+
+; CHECK: error: memory output constraint 'm' must be indirect
+define i64 @direct_m_output() {
+ %v = call i64 asm "", "=m"()
+ ret i64 %v
+}
+
+; CHECK: error: memory output constraint '{{[mo]}}' must be indirect
+define i64 @direct_mo_output() {
+ %v = call i64 asm "", "=mo"()
+ ret i64 %v
+}
diff --git a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output.ll b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output.ll
new file mode 100644
index 0000000000000..e4bf1eba56e4b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output.ll
@@ -0,0 +1,36 @@
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+
+define i64 @direct_rm_output() {
+; CHECK-LABEL: direct_rm_output:
+; CHECK: # %bb.0:
+; CHECK-NEXT: #APP
+; CHECK-NEXT: movq $42, %rax
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: retq
+ %v = call i64 asm "movq $$42, $0", "=rm"()
+ ret i64 %v
+}
+
+define i64 @direct_rm_output_used(i64 %x) {
+; CHECK-LABEL: direct_rm_output_used:
+; CHECK: # %bb.0:
+; CHECK-NEXT: #APP
+; CHECK-NEXT: movq $42, %rax
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: addq %rdi, %rax
+; CHECK-NEXT: retq
+ %v = call i64 asm "movq $$42, $0", "=rm"()
+ %s = add i64 %v, %x
+ ret i64 %s
+}
+
+define void @indirect_m_output(ptr %p) {
+; CHECK-LABEL: indirect_m_output:
+; CHECK: # %bb.0:
+; CHECK-NEXT: #APP
+; CHECK-NEXT: movq $0, (%rdi)
+; CHECK-NEXT: #NO_APP
+; CHECK-NEXT: retq
+ call void asm "movq $$0, $0", "=*m"(ptr elementtype(i64) %p)
+ ret void
+}
>From abc5ebff372ad1242cb78e3ff6879980079eec61 Mon Sep 17 00:00:00 2001
From: workwilson <ogbonnachibuoyim12 at gmail.com>
Date: Tue, 14 Jul 2026 19:31:38 +0800
Subject: [PATCH 2/3] Use constraint priority instead of erase_if for direct
memory outputs
---
.../CodeGen/SelectionDAG/TargetLowering.cpp | 40 +++++++++----------
1 file changed, 19 insertions(+), 21 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index 5cada45bd7d2f..79be6df2a0a1c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -6276,21 +6276,30 @@ TargetLowering::ParseConstraints(const DataLayout &DL,
/// over another, for the purpose of sorting them. Immediates are almost always
/// preferrable (when they can be emitted). A higher return value means a
/// stronger preference for one constraint type relative to another.
+///
+/// A direct output is made available as the asm's return value and consumes no
+/// call argument, so there is no address for the asm to write through. Memory
+/// and address constraints can therefore never be honored for one, and are
+/// ranked below registers so that any other alternative is preferred (e.g. "r"
+/// in "=rm"). If such a constraint is the only one on offer (e.g. "=m"), it is
+/// still selected, and lowering diagnoses it.
+///
/// FIXME: We should prefer registers over memory but doing so may lead to
/// unrecoverable register exhaustion later.
/// https://github.com/llvm/llvm-project/issues/20571
-static unsigned getConstraintPiority(TargetLowering::ConstraintType CT) {
+static unsigned getConstraintPiority(TargetLowering::ConstraintType CT,
+ bool IsDirectOutput) {
switch (CT) {
case TargetLowering::C_Immediate:
case TargetLowering::C_Other:
- return 4;
+ return 5;
case TargetLowering::C_Memory:
case TargetLowering::C_Address:
- return 3;
+ return IsDirectOutput ? 1 : 4;
case TargetLowering::C_RegisterClass:
- return 2;
+ return 3;
case TargetLowering::C_Register:
- return 1;
+ return 2;
case TargetLowering::C_Unknown:
return 0;
}
@@ -6416,23 +6425,12 @@ TargetLowering::ConstraintGroup TargetLowering::getConstraintPreferences(
Ret.emplace_back(Code, CType);
}
- // A direct output becomes the result of the asm rather than being written
- // through an address supplied by the caller, so it has to live in a
- // register. Prefer the other alternatives, if any (e.g. "r" in "=rm").
- // If memory is the only option (e.g. "=m"), keep it so that lowering can
- // diagnose the constraint; memory outputs must be spelled indirectly
- // (e.g. "=*m").
- if (OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect) {
- auto IsMemOrAddr = [](const ConstraintPair &P) {
- return P.second == TargetLowering::C_Memory ||
- P.second == TargetLowering::C_Address;
- };
- if (!llvm::all_of(Ret, IsMemOrAddr))
- llvm::erase_if(Ret, IsMemOrAddr);
- }
+ const bool IsDirectOutput =
+ OpInfo.Type == InlineAsm::isOutput && !OpInfo.isIndirect;
- llvm::stable_sort(Ret, [](ConstraintPair a, ConstraintPair b) {
- return getConstraintPiority(a.second) > getConstraintPiority(b.second);
+ llvm::stable_sort(Ret, [IsDirectOutput](ConstraintPair a, ConstraintPair b) {
+ return getConstraintPiority(a.second, IsDirectOutput) >
+ getConstraintPiority(b.second, IsDirectOutput);
});
return Ret;
>From 88e0e0a144347fb78c03429a529edbf3fd025c4a Mon Sep 17 00:00:00 2001
From: workwilson <ogbonnachibuoyim12 at gmail.com>
Date: Tue, 14 Jul 2026 19:35:27 +0800
Subject: [PATCH 3/3] Use explicit -global-isel=0 markers for non-GlobalISel
tests
---
.../CodeGen/AArch64/inline-asm-direct-mem-output-error.ll | 4 ++--
llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output.ll | 4 ++--
llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll | 2 +-
llvm/test/CodeGen/X86/inline-asm-direct-mem-output.ll | 2 +-
4 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
index 4f292d2d4105c..d0feb98a4034d 100644
--- a/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
+++ b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output-error.ll
@@ -1,5 +1,5 @@
-; RUN: not llc -mtriple=aarch64-unknown-linux-gnu < %s 2>&1 | FileCheck %s
-; RUN: not llc -mtriple=aarch64-unknown-linux-gnu -global-isel < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=aarch64-unknown-linux-gnu -global-isel=0 < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=aarch64-unknown-linux-gnu -global-isel=1 < %s 2>&1 | FileCheck %s
; CHECK: error: memory output constraint 'm' must be indirect
define i64 @direct_m_output() {
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output.ll b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output.ll
index 91baf51047b2e..20e9c61b7e5bf 100644
--- a/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output.ll
+++ b/llvm/test/CodeGen/AArch64/inline-asm-direct-mem-output.ll
@@ -1,5 +1,5 @@
-; RUN: llc -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s
-; RUN: llc -mtriple=aarch64-unknown-linux-gnu -global-isel < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu -global-isel=0 < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-unknown-linux-gnu -global-isel=1 < %s | FileCheck %s
define i64 @direct_rm_output() {
; CHECK-LABEL: direct_rm_output:
diff --git a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
index 121fa8729e4c8..b617e2ea9c7f0 100644
--- a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output-error.ll
@@ -1,4 +1,4 @@
-; RUN: not llc -mtriple=x86_64-unknown-linux-gnu < %s 2>&1 | FileCheck %s
+; RUN: not llc -mtriple=x86_64-unknown-linux-gnu -global-isel=0 < %s 2>&1 | FileCheck %s
; CHECK: error: memory output constraint 'm' must be indirect
define i64 @direct_m_output() {
diff --git a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output.ll b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output.ll
index e4bf1eba56e4b..4a85912f5ff6d 100644
--- a/llvm/test/CodeGen/X86/inline-asm-direct-mem-output.ll
+++ b/llvm/test/CodeGen/X86/inline-asm-direct-mem-output.ll
@@ -1,4 +1,4 @@
-; RUN: llc -mtriple=x86_64-unknown-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=x86_64-unknown-linux-gnu -global-isel=0 < %s | FileCheck %s
define i64 @direct_rm_output() {
; CHECK-LABEL: direct_rm_output:
More information about the llvm-commits
mailing list