[llvm] [X86] Don't use `rbp` when it's reserved (PR #146638)

[Brandt Bucher via llvm-commits]

https://github.com/brandtbucher created https://github.com/llvm/llvm-project/pull/146638

This fixes the x86 backend to properly reserve `rbp` when the `frame-pointer=reserved` option is used. Currently, this option is ignored.

Disassembly of the new test case before:

```s
pushq %rbp
pushq %rbx
pushq %rax
movl %esi, %ebx
movl %edi, %ebp
callq bar at PLT
movl %ebp, %edi
movl %ebx, %esi
callq bar at PLT
addq $8, %rsp
popq %rbx
popq %rbp
retq
```

...and after (`r14` is used as scratch space in place of `rbp`):

```s
pushq %r14
pushq %rbx
pushq %rax
movl %esi, %ebx
movl %edi, %r14d
callq bar at PLT
movl %r14d, %edi
movl %ebx, %esi
callq bar at PLT
addq $8, %rsp
popq %rbx
popq %r14
retq
```

Fixes #117178.

>From b18559450194169702a328b5a38e9ecebc4ee1db Mon Sep 17 00:00:00 2001
From: Brandt Bucher <brandtbucher at microsoft.com>
Date: Tue, 1 Jul 2025 22:09:41 -0700
Subject: [PATCH] Don't use rbp when it's reserved

---
 llvm/lib/Target/X86/X86RegisterInfo.cpp       |  2 +-
 .../CodeGen/X86/frame-pointer-reserved.ll     | 25 +++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/X86/frame-pointer-reserved.ll

diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index 9adac067aace8..83b11eede829e 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -562,7 +562,7 @@ BitVector X86RegisterInfo::getReservedRegs(const MachineFunction &MF) const {
     Reserved.set(SubReg);
 
   // Set the frame-pointer register and its aliases as reserved if needed.
-  if (TFI->hasFP(MF)) {
+  if (TFI->hasFP(MF) || MF.getTarget().Options.FramePointerIsReserved(MF)) {
     if (MF.getInfo<X86MachineFunctionInfo>()->getFPClobberedByInvoke())
       MF.getContext().reportError(
           SMLoc(),
diff --git a/llvm/test/CodeGen/X86/frame-pointer-reserved.ll b/llvm/test/CodeGen/X86/frame-pointer-reserved.ll
new file mode 100644
index 0000000000000..bf5887fe50508
--- /dev/null
+++ b/llvm/test/CodeGen/X86/frame-pointer-reserved.ll
@@ -0,0 +1,25 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s
+
+declare void @bar(i32, i32)
+
+define void @foo(i32 %0, i32 %1) nounwind "frame-pointer"="reserved" {
+; CHECK-LABEL: foo:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    pushq %r14
+; CHECK-NEXT:    pushq %rbx
+; CHECK-NEXT:    pushq %rax
+; CHECK-NEXT:    movl %esi, %ebx
+; CHECK-NEXT:    movl %edi, %r14d
+; CHECK-NEXT:    callq bar at PLT
+; CHECK-NEXT:    movl %r14d, %edi
+; CHECK-NEXT:    movl %ebx, %esi
+; CHECK-NEXT:    callq bar at PLT
+; CHECK-NEXT:    addq $8, %rsp
+; CHECK-NEXT:    popq %rbx
+; CHECK-NEXT:    popq %r14
+; CHECK-NEXT:    retq
+  call void @bar(i32 %0, i32 %1)
+  call void @bar(i32 %0, i32 %1)
+  ret void
+}