[llvm] 8d632c0 - [X86] Don't clobber x87 return values when zeroing call-used registers (#211129)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Jul 28 06:45:34 PDT 2026
Author: Akshay K
Date: 2026-07-28T21:45:28+08:00
New Revision: 8d632c0a8eef06b7be509d10a6d87c9bd814a10e
URL: https://github.com/llvm/llvm-project/commit/8d632c0a8eef06b7be509d10a6d87c9bd814a10e
DIFF: https://github.com/llvm/llvm-project/commit/8d632c0a8eef06b7be509d10a6d87c9bd814a10e.diff
LOG: [X86] Don't clobber x87 return values when zeroing call-used registers (#211129)
### Summary
`-fzero-call-used-regs=all` / `zero_call_used_regs("all")`
unconditionally push eight `fldz` onto the x87 stack before returning.
When a value is live in `ST0` (`long double`) or `ST0:ST1` (`_Complex
long double`), the pushes overflow the 8-slot x87 stack and the return
value becomes an indefinite NaN. Reproduces on x86-64 at every
optimization level, via both the flag and the attribute.
```c++
__attribute__((zero_call_used_regs("all")))
long double g(long double a) { return a + 1; }
// g(1.0L) returns nan, expected 2.0
```
### Root cause
The x87 return value lives in `ST0` (and `ST1` for `_Complex long
double`), but the FP stackifier's `handleReturn` deletes the `RET`'s FP
register operands after stackification (the `FP0`-`FP6` pseudos no
longer exist) without recording that the top of the x87 stack is live.
The scrub in `emitZeroCallUsedRegs` therefore has nothing telling it
`ST0` is occupied, so it pushes a zero over the return value.
### Fix
`handleReturn` now re-records the returned values as implicit
`ST0`/`ST1` uses on the return instruction — the same way every other
return register (`RAX`, `XMM0`, ...) is modeled — and
`emitZeroCallUsedRegs` pushes only `8 - live` zeros, filling the dead
slots and leaving the return value intact. This matches GCC.
Note: the implicit `ST0`/`ST1` operands are recorded on every
x87-returning `RET`, not only under `zero-call-used-regs`. One
consequence is that a `RET` now satisfies `isX87Instruction`, so
`X86InsertX87Wait` is adjusted so a return no longer suppresses the
strict-FP `wait` (a return performs no exception sync).
### Secondary fix (i386)
The old count was hardcoded `ST.is64Bit() ? 8 : 7`. On i386, a function
with no x87 value live at the return (e.g. an integer return) cleared
only seven of the eight physical x87 registers, so `"all"` silently left
one call-used register uncleared. The unified `8 - live` formula fixes
this too: i386 with no x87 return now pushes 8 (previously 7).
### Tests
`zero-call-used-regs.ll` adds `all_x87_return` (`x86_fp80`, 1 live -> 7
pushes), `all_x87_complex_return` (`_Complex long double`, 2 live -> 6
pushes), and `all_no_x87_live` (no x87 live -> 8 pushes, guarding the
i386 secondary fix). The full X86 CodeGen suite passes.
Fixes #211064
Assisted-by: Cursor
Added:
Modified:
llvm/lib/Target/X86/X86FloatingPoint.cpp
llvm/lib/Target/X86/X86FrameLowering.cpp
llvm/lib/Target/X86/X86InsertX87Wait.cpp
llvm/test/CodeGen/X86/zero-call-used-regs.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/X86/X86FloatingPoint.cpp b/llvm/lib/Target/X86/X86FloatingPoint.cpp
index 8b89058c31201..36e8ba45113a8 100644
--- a/llvm/lib/Target/X86/X86FloatingPoint.cpp
+++ b/llvm/lib/Target/X86/X86FloatingPoint.cpp
@@ -1135,6 +1135,15 @@ void FPS::handleReturn(MachineBasicBlock::iterator &I) {
--e;
}
+ // Record the values returned in ST0/ST1 as implicit uses so that later
+ // passes can tell they are live at the return.
+ if (FirstFPRegOp != ~0U)
+ MI.addOperand(MachineOperand::CreateReg(X86::ST0, /*isDef=*/false,
+ /*isImp=*/true));
+ if (SecondFPRegOp != ~0U)
+ MI.addOperand(MachineOperand::CreateReg(X86::ST1, /*isDef=*/false,
+ /*isImp=*/true));
+
// We may have been carrying spurious live-ins, so make sure only the
// returned registers are left live.
adjustLiveRegs(LiveMask, MI);
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index 942e014c64b66..8ce30b1c4e358 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -620,12 +620,18 @@ void X86FrameLowering::emitZeroCallUsedRegs(BitVector RegsToZero,
// Zero out FP stack if referenced. Do this outside of the loop below so that
// it's done only once.
- const X86Subtarget &ST = MF.getSubtarget<X86Subtarget>();
for (MCRegister Reg : RegsToZero.set_bits()) {
if (!X86::RFP80RegClass.contains(Reg))
continue;
- unsigned NumFPRegs = ST.is64Bit() ? 8 : 7;
+ // Do not push zeros over x87 return values. X86FloatingPoint records
+ // returned values as implicit ST0/ST1 uses on the return instruction.
+ unsigned NumFPRegs = 8;
+ if (MBBI->hasRegisterImplicitUseOperand(X86::ST0))
+ --NumFPRegs;
+ if (MBBI->hasRegisterImplicitUseOperand(X86::ST1))
+ --NumFPRegs;
+
for (unsigned i = 0; i != NumFPRegs; ++i)
BuildMI(MBB, MBBI, DL, TII.get(X86::LD_F0));
diff --git a/llvm/lib/Target/X86/X86InsertX87Wait.cpp b/llvm/lib/Target/X86/X86InsertX87Wait.cpp
index 751e5badb9a98..aea5c0681daac 100644
--- a/llvm/lib/Target/X86/X86InsertX87Wait.cpp
+++ b/llvm/lib/Target/X86/X86InsertX87Wait.cpp
@@ -108,12 +108,15 @@ static bool insertWaitInstruction(MachineFunction &MF) {
// If the following instruction is an X87 instruction that performs the
// wait operation itself, we can omit inserting wait. Skip
// meta-instructions so the decision is independent of debug info, and
- // keep the wait for non-waiting (FN-prefixed) successors.
+ // keep the wait for non-waiting (FN-prefixed) successors. A return may
+ // carry implicit ST uses but performs no exception sync, so it never
+ // makes the wait redundant.
MachineBasicBlock::iterator AfterMI = std::next(MI);
MachineBasicBlock::iterator NextMI = AfterMI;
while (NextMI != MBB.end() && NextMI->isMetaInstruction())
++NextMI;
- if (NextMI != MBB.end() && X86::isX87Instruction(*NextMI) &&
+ if (NextMI != MBB.end() && !NextMI->isReturn() &&
+ X86::isX87Instruction(*NextMI) &&
classifyX87ControlInstruction(NextMI->getOpcode()) !=
X87ControlKind::NonWaiting)
continue;
diff --git a/llvm/test/CodeGen/X86/zero-call-used-regs.ll b/llvm/test/CodeGen/X86/zero-call-used-regs.ll
index bc59796389e8e..613315b797924 100644
--- a/llvm/test/CodeGen/X86/zero-call-used-regs.ll
+++ b/llvm/test/CodeGen/X86/zero-call-used-regs.ll
@@ -173,6 +173,8 @@ define dso_local i32 @all(i32 returned %x) local_unnamed_addr #0 "zero-call-used
; I386-NEXT: fldz
; I386-NEXT: fldz
; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fstp %st(0)
; I386-NEXT: fstp %st(0)
; I386-NEXT: fstp %st(0)
; I386-NEXT: fstp %st(0)
@@ -255,4 +257,261 @@ define dso_local void @tailcall(ptr %p) local_unnamed_addr #0 "zero-call-used-re
ret void
}
+; The x87 scrub must not push zeros over a return value live in ST0.
+define dso_local x86_fp80 @all_x87_return(x86_fp80 %x) local_unnamed_addr #0 "zero-call-used-regs"="all" {
+; I386-LABEL: all_x87_return:
+; I386: # %bb.0: # %entry
+; I386-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NEXT: fld1
+; I386-NEXT: faddp %st, %st(1)
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: xorl %eax, %eax
+; I386-NEXT: xorl %ecx, %ecx
+; I386-NEXT: xorl %edx, %edx
+; I386-NEXT: xorps %xmm0, %xmm0
+; I386-NEXT: xorps %xmm1, %xmm1
+; I386-NEXT: xorps %xmm2, %xmm2
+; I386-NEXT: xorps %xmm3, %xmm3
+; I386-NEXT: xorps %xmm4, %xmm4
+; I386-NEXT: xorps %xmm5, %xmm5
+; I386-NEXT: xorps %xmm6, %xmm6
+; I386-NEXT: xorps %xmm7, %xmm7
+; I386-NEXT: retl
+;
+; X86-64-LABEL: all_x87_return:
+; X86-64: # %bb.0: # %entry
+; X86-64-NEXT: fldt {{[0-9]+}}(%rsp)
+; X86-64-NEXT: fld1
+; X86-64-NEXT: faddp %st, %st(1)
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: xorl %eax, %eax
+; X86-64-NEXT: xorl %ecx, %ecx
+; X86-64-NEXT: xorl %edi, %edi
+; X86-64-NEXT: xorl %edx, %edx
+; X86-64-NEXT: xorl %esi, %esi
+; X86-64-NEXT: xorl %r8d, %r8d
+; X86-64-NEXT: xorl %r9d, %r9d
+; X86-64-NEXT: xorl %r10d, %r10d
+; X86-64-NEXT: xorl %r11d, %r11d
+; X86-64-NEXT: xorps %xmm0, %xmm0
+; X86-64-NEXT: xorps %xmm1, %xmm1
+; X86-64-NEXT: xorps %xmm2, %xmm2
+; X86-64-NEXT: xorps %xmm3, %xmm3
+; X86-64-NEXT: xorps %xmm4, %xmm4
+; X86-64-NEXT: xorps %xmm5, %xmm5
+; X86-64-NEXT: xorps %xmm6, %xmm6
+; X86-64-NEXT: xorps %xmm7, %xmm7
+; X86-64-NEXT: xorps %xmm8, %xmm8
+; X86-64-NEXT: xorps %xmm9, %xmm9
+; X86-64-NEXT: xorps %xmm10, %xmm10
+; X86-64-NEXT: xorps %xmm11, %xmm11
+; X86-64-NEXT: xorps %xmm12, %xmm12
+; X86-64-NEXT: xorps %xmm13, %xmm13
+; X86-64-NEXT: xorps %xmm14, %xmm14
+; X86-64-NEXT: xorps %xmm15, %xmm15
+; X86-64-NEXT: retq
+entry:
+ %add = fadd x86_fp80 %x, 0xK3FFF8000000000000000
+ ret x86_fp80 %add
+}
+
+; Same for a two-value (_Complex long double) return live in ST0:ST1.
+define dso_local { x86_fp80, x86_fp80 } @all_x87_complex_return(x86_fp80 %re, x86_fp80 %im) local_unnamed_addr #0 "zero-call-used-regs"="all" {
+; I386-LABEL: all_x87_complex_return:
+; I386: # %bb.0: # %entry
+; I386-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NEXT: fld1
+; I386-NEXT: fadd %st, %st(1)
+; I386-NEXT: faddp %st, %st(2)
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: xorl %eax, %eax
+; I386-NEXT: xorl %ecx, %ecx
+; I386-NEXT: xorl %edx, %edx
+; I386-NEXT: xorps %xmm0, %xmm0
+; I386-NEXT: xorps %xmm1, %xmm1
+; I386-NEXT: xorps %xmm2, %xmm2
+; I386-NEXT: xorps %xmm3, %xmm3
+; I386-NEXT: xorps %xmm4, %xmm4
+; I386-NEXT: xorps %xmm5, %xmm5
+; I386-NEXT: xorps %xmm6, %xmm6
+; I386-NEXT: xorps %xmm7, %xmm7
+; I386-NEXT: retl
+;
+; X86-64-LABEL: all_x87_complex_return:
+; X86-64: # %bb.0: # %entry
+; X86-64-NEXT: fldt {{[0-9]+}}(%rsp)
+; X86-64-NEXT: fldt {{[0-9]+}}(%rsp)
+; X86-64-NEXT: fld1
+; X86-64-NEXT: fadd %st, %st(1)
+; X86-64-NEXT: faddp %st, %st(2)
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: xorl %eax, %eax
+; X86-64-NEXT: xorl %ecx, %ecx
+; X86-64-NEXT: xorl %edi, %edi
+; X86-64-NEXT: xorl %edx, %edx
+; X86-64-NEXT: xorl %esi, %esi
+; X86-64-NEXT: xorl %r8d, %r8d
+; X86-64-NEXT: xorl %r9d, %r9d
+; X86-64-NEXT: xorl %r10d, %r10d
+; X86-64-NEXT: xorl %r11d, %r11d
+; X86-64-NEXT: xorps %xmm0, %xmm0
+; X86-64-NEXT: xorps %xmm1, %xmm1
+; X86-64-NEXT: xorps %xmm2, %xmm2
+; X86-64-NEXT: xorps %xmm3, %xmm3
+; X86-64-NEXT: xorps %xmm4, %xmm4
+; X86-64-NEXT: xorps %xmm5, %xmm5
+; X86-64-NEXT: xorps %xmm6, %xmm6
+; X86-64-NEXT: xorps %xmm7, %xmm7
+; X86-64-NEXT: xorps %xmm8, %xmm8
+; X86-64-NEXT: xorps %xmm9, %xmm9
+; X86-64-NEXT: xorps %xmm10, %xmm10
+; X86-64-NEXT: xorps %xmm11, %xmm11
+; X86-64-NEXT: xorps %xmm12, %xmm12
+; X86-64-NEXT: xorps %xmm13, %xmm13
+; X86-64-NEXT: xorps %xmm14, %xmm14
+; X86-64-NEXT: xorps %xmm15, %xmm15
+; X86-64-NEXT: retq
+entry:
+ %add.re = fadd x86_fp80 %re, 0xK3FFF8000000000000000
+ %add.im = fadd x86_fp80 %im, 0xK3FFF8000000000000000
+ %r0 = insertvalue { x86_fp80, x86_fp80 } poison, x86_fp80 %add.re, 0
+ %r1 = insertvalue { x86_fp80, x86_fp80 } %r0, x86_fp80 %add.im, 1
+ ret { x86_fp80, x86_fp80 } %r1
+}
+
+; No x87 value is live at the return, so all eight physical x87 registers must
+; be zeroed. On i386 the old hardcoded count of 7 pushed one fldz too few,
+; leaving a call-used x87 register unzeroed under "all".
+define dso_local i32 @all_no_x87_live(i32 %x) local_unnamed_addr #0 "zero-call-used-regs"="all" {
+; I386-LABEL: all_no_x87_live:
+; I386: # %bb.0: # %entry
+; I386-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NEXT: incl %eax
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fldz
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: fstp %st(0)
+; I386-NEXT: xorl %ecx, %ecx
+; I386-NEXT: xorl %edx, %edx
+; I386-NEXT: xorps %xmm0, %xmm0
+; I386-NEXT: xorps %xmm1, %xmm1
+; I386-NEXT: xorps %xmm2, %xmm2
+; I386-NEXT: xorps %xmm3, %xmm3
+; I386-NEXT: xorps %xmm4, %xmm4
+; I386-NEXT: xorps %xmm5, %xmm5
+; I386-NEXT: xorps %xmm6, %xmm6
+; I386-NEXT: xorps %xmm7, %xmm7
+; I386-NEXT: retl
+;
+; X86-64-LABEL: all_no_x87_live:
+; X86-64: # %bb.0: # %entry
+; X86-64-NEXT: # kill: def $edi killed $edi def $rdi
+; X86-64-NEXT: leal 1(%rdi), %eax
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fldz
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: fstp %st(0)
+; X86-64-NEXT: xorl %ecx, %ecx
+; X86-64-NEXT: xorl %edi, %edi
+; X86-64-NEXT: xorl %edx, %edx
+; X86-64-NEXT: xorl %esi, %esi
+; X86-64-NEXT: xorl %r8d, %r8d
+; X86-64-NEXT: xorl %r9d, %r9d
+; X86-64-NEXT: xorl %r10d, %r10d
+; X86-64-NEXT: xorl %r11d, %r11d
+; X86-64-NEXT: xorps %xmm0, %xmm0
+; X86-64-NEXT: xorps %xmm1, %xmm1
+; X86-64-NEXT: xorps %xmm2, %xmm2
+; X86-64-NEXT: xorps %xmm3, %xmm3
+; X86-64-NEXT: xorps %xmm4, %xmm4
+; X86-64-NEXT: xorps %xmm5, %xmm5
+; X86-64-NEXT: xorps %xmm6, %xmm6
+; X86-64-NEXT: xorps %xmm7, %xmm7
+; X86-64-NEXT: xorps %xmm8, %xmm8
+; X86-64-NEXT: xorps %xmm9, %xmm9
+; X86-64-NEXT: xorps %xmm10, %xmm10
+; X86-64-NEXT: xorps %xmm11, %xmm11
+; X86-64-NEXT: xorps %xmm12, %xmm12
+; X86-64-NEXT: xorps %xmm13, %xmm13
+; X86-64-NEXT: xorps %xmm14, %xmm14
+; X86-64-NEXT: xorps %xmm15, %xmm15
+; X86-64-NEXT: retq
+entry:
+ %add = add i32 %x, 1
+ ret i32 %add
+}
+
attributes #0 = { mustprogress nofree norecurse nosync nounwind readnone uwtable willreturn "frame-pointer"="none" "min-legal-vector-width"="0" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+cx8,+fxsr,+mmx,+sse,+sse2,+x87" "tune-cpu"="generic" }
More information about the llvm-commits
mailing list