[llvm] 6526e5b - [X86] Fix FP `stackifier` crash on invalid `x87` inline `asm` constraints (#221881)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 14 23:54:56 PDT 2026
Author: Akash Manna
Date: 2026-09-15T14:54:50+08:00
New Revision: 6526e5bc64b15efd1c972f6035f34a432637948e
URL: https://github.com/llvm/llvm-project/commit/6526e5bc64b15efd1c972f6035f34a432637948e
DIFF: https://github.com/llvm/llvm-project/commit/6526e5bc64b15efd1c972f6035f34a432637948e.diff
LOG: [X86] Fix FP `stackifier` crash on invalid `x87` inline `asm` constraints (#221881)
Fixes #149371
When an x87 inline asm uses the stack registers in a layout the FP
stackifier can't model, for example an input tied to `st(1)` while
`st(0)` and `st(1)` are both outputs, the pass reports "fixed input regs
must be last on the x87 stack" and then keeps going as if nothing
happened. It simulates the pops and pushes with counts taken from masks
it has just found to be non-contiguous, so the input never leaves the
stack model and the same register gets pushed again as an output. The
model now holds a duplicate entry, and the next `adjustLiveRegs` trips
"Live count mismatch". The odd `R0` constraint in the report isn't the
culprit, by the way; it resolves to a plain integer register and leaves
the tied `1` as the only x87 input.
Once any of these errors has been reported, the asm is no longer
simulated. The pass pops the inputs the asm kills and pushes its
outputs, which keeps the stack model consistent with what the register
allocator expects, and then lets compilation fail normally with the
diagnostic. The old trick of rounding the output mask up to a contiguous
range is gone too, since it only papered over one of the four cases. The
new test covers all four diagnostics plus the two-asm sequence that used
to assert.
LLM tools were used for this contribution. I've reviewed, built, and
tested the change myself before pushing to GitHub.
Added:
llvm/test/CodeGen/X86/pr149371.ll
Modified:
llvm/lib/Target/X86/X86FloatingPoint.cpp
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86FloatingPoint.cpp b/llvm/lib/Target/X86/X86FloatingPoint.cpp
index 36e8ba45113a8..5364606689d5a 100644
--- a/llvm/lib/Target/X86/X86FloatingPoint.cpp
+++ b/llvm/lib/Target/X86/X86FloatingPoint.cpp
@@ -1664,41 +1664,31 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) {
}
}
- if (STUses && !isMask_32(STUses))
+ bool Malformed = false;
+ if (STUses && !isMask_32(STUses)) {
MI.emitGenericError("fixed input regs must be last on the x87 stack");
- unsigned NumSTUses = llvm::countr_one(STUses);
+ Malformed = true;
+ }
// Defs must be contiguous from the stack top. ST0-STn.
if (STDefs && !isMask_32(STDefs)) {
MI.emitGenericError("output regs must be last on the x87 stack");
- STDefs = NextPowerOf2(STDefs) - 1;
+ Malformed = true;
}
- unsigned NumSTDefs = llvm::countr_one(STDefs);
// So must the clobbered stack slots. ST0-STm, m >= n.
- if (STClobbers && !isMask_32(STDefs | STClobbers))
+ if (STClobbers && !isMask_32(STDefs | STClobbers)) {
MI.emitGenericError("clobbers must be last on the x87 stack");
+ Malformed = true;
+ }
// Popped inputs are the ones that are also clobbered or defined.
unsigned STPopped = STUses & (STDefs | STClobbers);
- if (STPopped && !isMask_32(STPopped))
+ if (STPopped && !isMask_32(STPopped)) {
MI.emitGenericError(
"implicitly popped regs must be last on the x87 stack");
- unsigned NumSTPopped = llvm::countr_one(STPopped);
-
- LLVM_DEBUG(dbgs() << "Asm uses " << NumSTUses << " fixed regs, pops "
- << NumSTPopped << ", and defines " << NumSTDefs
- << " regs.\n");
-
-#ifndef NDEBUG
- // If any input operand uses constraint "f", all output register
- // constraints must be early-clobber defs.
- for (unsigned I = 0, E = MI.getNumOperands(); I < E; ++I)
- if (FRegIdx.count(I)) {
- assert((1 << getFPReg(MI.getOperand(I)) & STDefs) == 0 &&
- "Operands with constraint \"f\" cannot overlap with defs");
- }
-#endif
+ Malformed = true;
+ }
// Collect all FP registers (register operands with constraints "t", "u",
// and "f") to kill afer the instruction.
@@ -1715,6 +1705,60 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) {
FPKills |= 1U << FPReg;
}
+ auto RewriteFPRegs = [&]() {
+ for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
+ MachineOperand &Op = MI.getOperand(i);
+ if (!Op.isReg() || Op.getReg() < X86::FP0 || Op.getReg() > X86::FP6)
+ continue;
+
+ unsigned FPReg = getFPReg(Op);
+
+ if (FRegIdx.count(i))
+ // Operand with constraint "f".
+ Op.setReg(getSTReg(FPReg));
+ else
+ // Operand with a single register class constraint ("t" or "u").
+ Op.setReg(X86::ST0 + FPReg);
+ }
+ };
+
+ if (Malformed) {
+ // Don't simulate a malformed asm. Just keep the stack model consistent:
+ // pop the killed inputs and push the outputs.
+ RewriteFPRegs();
+ while (FPKills) {
+ unsigned FPReg = llvm::countr_zero(FPKills);
+ if (isLive(FPReg))
+ freeStackSlotBefore(Inst, FPReg);
+ FPKills &= ~(1U << FPReg);
+ }
+ while (STDefs) {
+ unsigned FPReg = llvm::countr_zero(STDefs);
+ if (!isLive(FPReg)) {
+ BuildMI(*MBB, Inst, MI.getDebugLoc(), TII->get(X86::LD_F0));
+ pushReg(FPReg);
+ }
+ STDefs &= ~(1U << FPReg);
+ }
+ return;
+ }
+
+ unsigned NumSTUses = llvm::countr_one(STUses);
+ unsigned NumSTDefs = llvm::countr_one(STDefs);
+ unsigned NumSTPopped = llvm::countr_one(STPopped);
+
+ LLVM_DEBUG(dbgs() << "Asm uses " << NumSTUses << " fixed regs, pops "
+ << NumSTPopped << ", and defines " << NumSTDefs
+ << " regs.\n");
+
+ // If any input operand uses constraint "f", all output register
+ // constraints must be early-clobber defs.
+ assert(llvm::none_of(FRegIdx,
+ [&](unsigned I) {
+ return STDefs & (1u << getFPReg(MI.getOperand(I)));
+ }) &&
+ "Operands with constraint \"f\" cannot overlap with defs");
+
// Do not include registers that are implicitly popped by defs/clobbers.
FPKills &= ~(STDefs | STClobbers);
@@ -1731,20 +1775,7 @@ void FPS::handleSpecialFP(MachineBasicBlock::iterator &Inst) {
});
// With the stack layout fixed, rewrite the FP registers.
- for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
- MachineOperand &Op = MI.getOperand(i);
- if (!Op.isReg() || Op.getReg() < X86::FP0 || Op.getReg() > X86::FP6)
- continue;
-
- unsigned FPReg = getFPReg(Op);
-
- if (FRegIdx.count(i))
- // Operand with constraint "f".
- Op.setReg(getSTReg(FPReg));
- else
- // Operand with a single register class constraint ("t" or "u").
- Op.setReg(X86::ST0 + FPReg);
- }
+ RewriteFPRegs();
// Simulate the inline asm popping its inputs and pushing its outputs.
StackTop -= NumSTPopped;
diff --git a/llvm/test/CodeGen/X86/pr149371.ll b/llvm/test/CodeGen/X86/pr149371.ll
new file mode 100644
index 0000000000000..5495df2f4e71f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/pr149371.ll
@@ -0,0 +1,38 @@
+; RUN: not llc -mtriple=x86_64 < %s 2>&1 | FileCheck %s
+
+; Inline asm whose x87 register constraints don't describe a valid stack
+; layout must be diagnosed without crashing the FP stackifier afterwards.
+
+; CHECK: error: {{.*}}: fixed input regs must be last on the x87 stack
+define void @fixed_input(double %y) {
+ call void asm sideeffect "", "{st(1)},~{dirflag},~{fpsr},~{flags}"(double %y)
+ ret void
+}
+
+; CHECK: error: {{.*}}: output regs must be last on the x87 stack
+define double @output_regs() {
+ %r = call double asm sideeffect "", "={st(1)},~{dirflag},~{fpsr},~{flags}"()
+ ret double %r
+}
+
+; CHECK: error: {{.*}}: clobbers must be last on the x87 stack
+define double @clobbers() {
+ %r = call double asm sideeffect "", "={st},~{st(2)},~{dirflag},~{fpsr},~{flags}"()
+ ret double %r
+}
+
+; PR149371: the only x87 input is tied to st(1) while st(0) and st(1) are
+; both outputs. Simulating the pops and pushes for this layout used to leave
+; a stale entry in the stack model, and the second asm then tripped
+; "Live count mismatch" at the return.
+; CHECK: error: {{.*}}: fixed input regs must be last on the x87 stack
+; CHECK-NEXT: error: {{.*}}: implicitly popped regs must be last on the x87 stack
+; CHECK-NEXT: error: {{.*}}: fixed input regs must be last on the x87 stack
+; CHECK-NEXT: error: {{.*}}: implicitly popped regs must be last on the x87 stack
+; CHECK-NOT: Assertion
+; CHECK-NOT: LLVM ERROR
+define void @popped(double %x, double %y) {
+ %r1 = call { double, double } asm sideeffect "", "={st},={st(1)},R0,1,~{dirflag},~{fpsr},~{flags}"(double %x, double %y)
+ %r2 = call { double, double } asm sideeffect "", "={st},={st(1)},R0,1,~{dirflag},~{fpsr},~{flags}"(double %x, double %y)
+ ret void
+}
More information about the llvm-commits
mailing list