[llvm] [X86] Use R10 as a fallback for saturated indirect musttail calls (PR #208246)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Jul 8 08:52:07 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: AxXxB
<details>
<summary>Changes</summary>
SysV excludes R10 from the ordinary indirect tail-call target register class
because it is used for nest parameters. For calling conventions such as
preserve_nonecc, a musttail call can still saturate every ordinary target
register with outgoing arguments, leaving no register available for the
indirect target.
Use R10 as a fallback only when the ordinary GR64_TC path has no available
candidate and the outgoing call assignments do not use R10, including nest
arguments. Keep the fallback on a separate TC_RETURN_R10 pseudo path so
ordinary indirect tail calls continue to use the existing register class.
Tests cover the saturated preserve_nonecc case, the existing free-register
path, and a case where the indirect target is also an outgoing argument.
Tested with `ninja -C build -j8 llc opt llvm-mc llvm-objdump llvm-readobj FileCheck`
and `build/bin/llvm-lit -q llvm/test/CodeGen/X86`.
---
Full diff: https://github.com/llvm/llvm-project/pull/208246.diff
13 Files Affected:
- (modified) llvm/lib/Target/X86/X86AsmPrinter.cpp (+4-1)
- (modified) llvm/lib/Target/X86/X86ExpandPseudo.cpp (+7-3)
- (modified) llvm/lib/Target/X86/X86FrameLowering.cpp (+1)
- (modified) llvm/lib/Target/X86/X86ISelLowering.cpp (+2)
- (modified) llvm/lib/Target/X86/X86ISelLoweringCall.cpp (+58-1)
- (modified) llvm/lib/Target/X86/X86InstrCompiler.td (+4)
- (modified) llvm/lib/Target/X86/X86InstrControl.td (+7)
- (modified) llvm/lib/Target/X86/X86InstrFragments.td (+2)
- (modified) llvm/lib/Target/X86/X86InstrInfo.cpp (+1)
- (modified) llvm/lib/Target/X86/X86MCInstLower.cpp (+10)
- (modified) llvm/lib/Target/X86/X86RegisterInfo.cpp (+1)
- (added) llvm/test/CodeGen/X86/preserve-nonecc-indirect-tailcall.ll (+36)
- (modified) llvm/test/CodeGen/X86/tailcall-ri64.ll (+1-1)
``````````diff
diff --git a/llvm/lib/Target/X86/X86AsmPrinter.cpp b/llvm/lib/Target/X86/X86AsmPrinter.cpp
index f22b89b9b6e96..4d155fb6853c0 100644
--- a/llvm/lib/Target/X86/X86AsmPrinter.cpp
+++ b/llvm/lib/Target/X86/X86AsmPrinter.cpp
@@ -488,11 +488,14 @@ static bool isIndirectBranchOrTailCall(const MachineInstr &MI) {
return MI.getDesc().isIndirectBranch() /*Make below code in a good shape*/ ||
Opc == X86::TAILJMPr || Opc == X86::TAILJMPm ||
Opc == X86::TAILJMPr64 || Opc == X86::TAILJMPm64 ||
+ Opc == X86::TAILJMPr64_R10 ||
Opc == X86::TCRETURNri || Opc == X86::TCRETURN_WIN64ri ||
Opc == X86::TCRETURN_HIPE32ri || Opc == X86::TCRETURNmi ||
Opc == X86::TCRETURN_WINmi64 || Opc == X86::TCRETURNri64 ||
+ Opc == X86::TCRETURNri64_R10 ||
Opc == X86::TCRETURNmi64 || Opc == X86::TCRETURNri64_ImpCall ||
- Opc == X86::TAILJMPr64_REX || Opc == X86::TAILJMPm64_REX;
+ Opc == X86::TAILJMPr64_REX || Opc == X86::TAILJMPr64_R10_REX ||
+ Opc == X86::TAILJMPm64_REX;
}
void X86AsmPrinter::emitBasicBlockEnd(const MachineBasicBlock &MBB) {
diff --git a/llvm/lib/Target/X86/X86ExpandPseudo.cpp b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
index 017686b50abcf..9a9745c467bbb 100644
--- a/llvm/lib/Target/X86/X86ExpandPseudo.cpp
+++ b/llvm/lib/Target/X86/X86ExpandPseudo.cpp
@@ -290,6 +290,7 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
case X86::TCRETURNdi64:
case X86::TCRETURNdi64cc:
case X86::TCRETURNri64:
+ case X86::TCRETURNri64_R10:
case X86::TCRETURNri64_ImpCall:
case X86::TCRETURNmi64:
case X86::TCRETURN_WINmi64: {
@@ -367,12 +368,15 @@ bool X86ExpandPseudoImpl::expandMI(MachineBasicBlock &MBB,
for (unsigned i = 0; i != X86::AddrNumOperands; ++i)
MIB.add(MBBI->getOperand(i));
} else if (Opcode == X86::TCRETURNri64 ||
+ Opcode == X86::TCRETURNri64_R10 ||
Opcode == X86::TCRETURNri64_ImpCall ||
Opcode == X86::TCRETURN_WIN64ri) {
JumpTarget.setIsKill();
- BuildMI(MBB, MBBI, DL,
- TII->get(IsX64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64))
- .add(JumpTarget);
+ unsigned TailJumpOp =
+ Opcode == X86::TCRETURNri64_R10
+ ? (IsX64 ? X86::TAILJMPr64_R10_REX : X86::TAILJMPr64_R10)
+ : (IsX64 ? X86::TAILJMPr64_REX : X86::TAILJMPr64);
+ BuildMI(MBB, MBBI, DL, TII->get(TailJumpOp)).add(JumpTarget);
} else {
assert(!IsX64 && "Win64 and UEFI64 require REX for indirect jumps.");
JumpTarget.setIsKill();
diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp
index a6e1d998c051a..d47d3b2553206 100644
--- a/llvm/lib/Target/X86/X86FrameLowering.cpp
+++ b/llvm/lib/Target/X86/X86FrameLowering.cpp
@@ -2483,6 +2483,7 @@ static bool isTailCallOpcode(unsigned Opc) {
return Opc == X86::TCRETURNri || Opc == X86::TCRETURN_WIN64ri ||
Opc == X86::TCRETURN_HIPE32ri || Opc == X86::TCRETURNdi ||
Opc == X86::TCRETURNmi || Opc == X86::TCRETURNri64 ||
+ Opc == X86::TCRETURNri64_R10 ||
Opc == X86::TCRETURNri64_ImpCall || Opc == X86::TCRETURNdi64 ||
Opc == X86::TCRETURNmi64 || Opc == X86::TCRETURN_WINmi64;
}
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index cc3aba9d431d6..83b422a419491 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -64476,7 +64476,9 @@ X86TargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
case X86::CALL64r_ImpCall:
case X86::CALL64r_NT:
case X86::TAILJMPr64:
+ case X86::TAILJMPr64_R10:
case X86::TAILJMPr64_REX:
+ case X86::TAILJMPr64_R10_REX:
assert(Target.isReg() && "Unexpected target operand for an indirect call");
Target.setIsRenamable(false);
TargetReg = Target.getReg();
diff --git a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
index 819e0a023c1c5..f57f27e05883d 100644
--- a/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
+++ b/llvm/lib/Target/X86/X86ISelLoweringCall.cpp
@@ -67,6 +67,45 @@ static bool shouldDisableArgRegFromCSR(CallingConv::ID CC) {
return CC == CallingConv::X86_RegCall;
}
+static bool canUseR10ForIndirectTailCallTarget(
+ MachineFunction &MF, const X86RegisterInfo *RegInfo,
+ ArrayRef<ISD::OutputArg> Outs, ArrayRef<CCValAssign> ArgLocs,
+ ArrayRef<std::pair<Register, SDValue>> RegsToPass) {
+ if (RegInfo->getReservedRegs(MF).test(X86::R10))
+ return false;
+
+ auto IsRegUsed = [&](MCPhysReg Reg) {
+ for (const auto &RegToPass : RegsToPass)
+ if (RegInfo->regsOverlap(RegToPass.first, Reg))
+ return true;
+
+ return false;
+ };
+
+ // Keep the existing GR64_TC path when it has any usable candidate. R10 is
+ // only a fallback for musttail calls whose outgoing register arguments occupy
+ // every ordinary indirect-tailcall target register.
+ for (MCPhysReg Reg : {X86::RAX, X86::RCX, X86::RDX, X86::RSI, X86::RDI,
+ X86::R8, X86::R9, X86::R11})
+ if (!RegInfo->getReservedRegs(MF).test(Reg) && !IsRegUsed(Reg))
+ return false;
+
+ for (const CCValAssign &VA : ArgLocs) {
+ unsigned ValNo = VA.getValNo();
+ if (ValNo < Outs.size() && Outs[ValNo].Flags.isNest())
+ return false;
+
+ if (VA.isRegLoc() && RegInfo->regsOverlap(VA.getLocReg(), X86::R10))
+ return false;
+ }
+
+ for (const auto &RegToPass : RegsToPass)
+ if (RegInfo->regsOverlap(RegToPass.first, X86::R10))
+ return false;
+
+ return true;
+}
+
static std::pair<MVT, unsigned>
handleMaskRegisterForCallingConv(unsigned NumElts, CallingConv::ID CC,
const X86Subtarget &Subtarget) {
@@ -2606,6 +2645,21 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
GA->getGlobal(), dl, GA->getValueType(0), 0, X86II::MO_NO_FLAG);
}
+ bool UseR10TailCallTarget = false;
+ if (Is64Bit && isTailCall && IsMustTail && IsIndirectCall && !IsWin64 &&
+ !IsNoTrackIndirectCall && !IsCFICall &&
+ !M->getModuleFlag("import-call-optimization") &&
+ Callee.getValueType() == MVT::i64)
+ UseR10TailCallTarget =
+ canUseR10ForIndirectTailCallTarget(MF, RegInfo, Outs, ArgLocs,
+ RegsToPass);
+
+ if (UseR10TailCallTarget) {
+ Chain = DAG.getCopyToReg(Chain, dl, X86::R10, Callee, InGlue);
+ InGlue = Chain.getValue(1);
+ Callee = DAG.getRegister(X86::R10, MVT::i64);
+ }
+
SmallVector<SDValue, 8> Ops;
if (!IsSibcall && isTailCall && !IsMustTail) {
@@ -2709,7 +2763,10 @@ X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
// function making a tail call to a function returning int.
MF.getFrameInfo().setHasTailCall();
auto Opcode =
- IsCFGuardCall ? X86ISD::TC_RETURN_GLOBALADDR : X86ISD::TC_RETURN;
+ UseR10TailCallTarget
+ ? X86ISD::TC_RETURN_R10
+ : (IsCFGuardCall ? X86ISD::TC_RETURN_GLOBALADDR
+ : X86ISD::TC_RETURN);
SDValue Ret = DAG.getNode(Opcode, dl, MVT::Other, Ops);
if (IsCFICall)
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 35dfdf2d0067c..4a90c21d31a67 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -1403,6 +1403,10 @@ def : Pat<(X86tcret ptr_rc_tailcall:$dst, timm:$off),
(TCRETURNri64 ptr_rc_tailcall:$dst, timm:$off)>,
Requires<[In64BitMode, IsNotWin64CCFunc, NotUseIndirectThunkCalls, ImportCallOptimizationDisabled]>;
+def : Pat<(X86tcret_r10 GR64_ArgRef:$dst, timm:$off),
+ (TCRETURNri64_R10 GR64_ArgRef:$dst, timm:$off)>,
+ Requires<[In64BitMode, IsNotWin64CCFunc, NotUseIndirectThunkCalls, ImportCallOptimizationDisabled]>;
+
def : Pat<(X86tcret GR64_TCW64:$dst, timm:$off),
(TCRETURN_WIN64ri GR64_TCW64:$dst, timm:$off)>,
Requires<[IsWin64CCFunc, NotUseIndirectThunkCalls, ImportCallOptimizationDisabled]>;
diff --git a/llvm/lib/Target/X86/X86InstrControl.td b/llvm/lib/Target/X86/X86InstrControl.td
index 1b2dbefce6f1c..369a33a4e9c3f 100644
--- a/llvm/lib/Target/X86/X86InstrControl.td
+++ b/llvm/lib/Target/X86/X86InstrControl.td
@@ -365,6 +365,9 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
def TCRETURNri64 : PseudoI<(outs),
(ins GR64_TC:$dst, i32imm:$offset),
[]>, Sched<[WriteJump]>;
+ def TCRETURNri64_R10 : PseudoI<(outs),
+ (ins GR64_ArgRef:$dst, i32imm:$offset),
+ []>, Sched<[WriteJump]>;
def TCRETURN_WIN64ri : PseudoI<(outs), (ins GR64_TCW64:$dst, i32imm:$offset),
[]>, Sched<[WriteJump]>;
@@ -386,6 +389,8 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
def TAILJMPr64 : PseudoI<(outs), (ins ptr_rc_tailcall:$dst),
[]>, Sched<[WriteJump]>;
+ def TAILJMPr64_R10 : PseudoI<(outs), (ins GR64_ArgRef:$dst),
+ []>, Sched<[WriteJump]>;
let mayLoad = 1 in
def TAILJMPm64 : PseudoI<(outs), (ins i64mem_TC:$dst),
@@ -395,6 +400,8 @@ let isCall = 1, isTerminator = 1, isReturn = 1, isBarrier = 1,
let hasREX_W = 1 in {
def TAILJMPr64_REX : PseudoI<(outs), (ins ptr_rc_tailcall:$dst),
[]>, Sched<[WriteJump]>;
+ def TAILJMPr64_R10_REX : PseudoI<(outs), (ins GR64_ArgRef:$dst),
+ []>, Sched<[WriteJump]>;
let mayLoad = 1 in
def TAILJMPm64_REX : PseudoI<(outs), (ins i64mem_TC:$dst),
diff --git a/llvm/lib/Target/X86/X86InstrFragments.td b/llvm/lib/Target/X86/X86InstrFragments.td
index c183849d4f575..2cadf8b8ce37a 100644
--- a/llvm/lib/Target/X86/X86InstrFragments.td
+++ b/llvm/lib/Target/X86/X86InstrFragments.td
@@ -375,6 +375,8 @@ def X86eh_sjlj_setup_dispatch : SDNode<"X86ISD::EH_SJLJ_SETUP_DISPATCH",
// the list of operands.
def X86tcret : SDNode<"X86ISD::TC_RETURN", SDT_X86TCRET,
[SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
+def X86tcret_r10 : SDNode<"X86ISD::TC_RETURN_R10", SDT_X86TCRET,
+ [SDNPHasChain, SDNPOptInGlue, SDNPVariadic]>;
// Pseudo for a tail call return to a global address that must be called via
// a memory address (i.e., not loaded into a register then called).
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 7ff2400d06d1d..8f033bfc381f5 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -3708,6 +3708,7 @@ bool X86InstrInfo::isUnconditionalTailCall(const MachineInstr &MI) const {
case X86::TCRETURNmi:
case X86::TCRETURNdi64:
case X86::TCRETURNri64:
+ case X86::TCRETURNri64_R10:
case X86::TCRETURNri64_ImpCall:
case X86::TCRETURNmi64:
return true;
diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp
index ad0946b4c3310..6f7d8a68a2529 100644
--- a/llvm/lib/Target/X86/X86MCInstLower.cpp
+++ b/llvm/lib/Target/X86/X86MCInstLower.cpp
@@ -379,12 +379,14 @@ static unsigned convertTailJumpOpcode(unsigned Opcode, bool IsLarge = false) {
Opcode = X86::JMP32m;
break;
case X86::TAILJMPr64:
+ case X86::TAILJMPr64_R10:
Opcode = X86::JMP64r;
break;
case X86::TAILJMPm64:
Opcode = X86::JMP64m;
break;
case X86::TAILJMPr64_REX:
+ case X86::TAILJMPr64_R10_REX:
Opcode = X86::JMP64r_REX;
break;
case X86::TAILJMPm64_REX:
@@ -483,7 +485,9 @@ void X86MCInstLower::Lower(const MachineInstr *MI, MCInst &OutMI) const {
// instruction.
case X86::TAILJMPr:
case X86::TAILJMPr64:
+ case X86::TAILJMPr64_R10:
case X86::TAILJMPr64_REX:
+ case X86::TAILJMPr64_R10_REX:
case X86::TAILJMPd:
assert(OutMI.getNumOperands() == 1 && "Unexpected number of operands!");
OutMI.setOpcode(convertTailJumpOpcode(OutMI.getOpcode()));
@@ -2367,6 +2371,7 @@ void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
case X86::TAILJMPd:
case X86::TAILJMPd_CC:
case X86::TAILJMPr64:
+ case X86::TAILJMPr64_R10:
case X86::TAILJMPm64:
case X86::TAILJMPd64_CC:
if (EnableImportCallOptimization)
@@ -2388,6 +2393,11 @@ void X86AsmPrinter::emitInstruction(const MachineInstr *MI) {
IsTailJump = true;
break;
+ case X86::TAILJMPr64_R10_REX:
+ OutStreamer->AddComment("TAILCALL");
+ IsTailJump = true;
+ break;
+
case X86::TAILJMPr64_REX: {
if (EnableImportCallOptimization) {
assert(MI->getOperand(0).getReg() == X86::RAX &&
diff --git a/llvm/lib/Target/X86/X86RegisterInfo.cpp b/llvm/lib/Target/X86/X86RegisterInfo.cpp
index ba137c7e7350d..c474ec87897ed 100644
--- a/llvm/lib/Target/X86/X86RegisterInfo.cpp
+++ b/llvm/lib/Target/X86/X86RegisterInfo.cpp
@@ -1069,6 +1069,7 @@ unsigned X86RegisterInfo::findDeadCallerSavedReg(
case X86::TCRETURNmi:
case X86::TCRETURNdi64:
case X86::TCRETURNri64:
+ case X86::TCRETURNri64_R10:
case X86::TCRETURNri64_ImpCall:
case X86::TCRETURNmi64:
case X86::TCRETURN_WINmi64:
diff --git a/llvm/test/CodeGen/X86/preserve-nonecc-indirect-tailcall.ll b/llvm/test/CodeGen/X86/preserve-nonecc-indirect-tailcall.ll
new file mode 100644
index 0000000000000..fe997355e992c
--- /dev/null
+++ b/llvm/test/CodeGen/X86/preserve-nonecc-indirect-tailcall.ll
@@ -0,0 +1,36 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -relocation-model=static | FileCheck %s
+
+ at target_ptr = external global ptr
+
+define preserve_nonecc void @dyn_tail(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, i64 %a16, i64 %a17, i64 %a18, i64 %a19, i64 %a20, i64 %a21, i64 %a22, i64 %a23, i64 %a24, i64 %a25, i64 %a26, i64 %a27, i64 %a28, i64 %a29, i64 %a30, i64 %a31, i64 %a32, i64 %a33, i64 %a34, i64 %a35) {
+; CHECK-LABEL: dyn_tail:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq target_ptr at GOTPCREL(%rip), %r10
+; CHECK-NEXT: movq (%r10), %r10
+; CHECK-NEXT: jmpq *%r10 # TAILCALL
+entry:
+ %target = load ptr, ptr @target_ptr, align 8
+ musttail call preserve_nonecc void %target(i64 %a0, i64 %a1, i64 %a2, i64 %a3, i64 %a4, i64 %a5, i64 %a6, i64 %a7, i64 %a8, i64 %a9, i64 %a10, i64 %a11, i64 %a12, i64 %a13, i64 %a14, i64 %a15, i64 %a16, i64 %a17, i64 %a18, i64 %a19, i64 %a20, i64 %a21, i64 %a22, i64 %a23, i64 %a24, i64 %a25, i64 %a26, i64 %a27, i64 %a28, i64 %a29, i64 %a30, i64 %a31, i64 %a32, i64 %a33, i64 %a34, i64 %a35)
+ ret void
+}
+
+define preserve_nonecc void @dyn_tail_free_regs() {
+; CHECK-LABEL: dyn_tail_free_regs:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq target_ptr at GOTPCREL(%rip), %rax
+; CHECK-NEXT: jmpq *(%rax) # TAILCALL
+entry:
+ %target = load ptr, ptr @target_ptr, align 8
+ musttail call preserve_nonecc void %target()
+ ret void
+}
+
+define preserve_nonecc void @dyn_tail_arg(ptr %target, i64 %a0) {
+; CHECK-LABEL: dyn_tail_arg:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %r12, %rax
+; CHECK-NEXT: jmpq *%rax # TAILCALL
+entry:
+ musttail call preserve_nonecc void %target(ptr %target, i64 %a0)
+ ret void
+}
diff --git a/llvm/test/CodeGen/X86/tailcall-ri64.ll b/llvm/test/CodeGen/X86/tailcall-ri64.ll
index 4a9b7701ae0a5..0bf7b65fd5734 100644
--- a/llvm/test/CodeGen/X86/tailcall-ri64.ll
+++ b/llvm/test/CodeGen/X86/tailcall-ri64.ll
@@ -4,7 +4,7 @@
; TAILJMPri64 should not receive "callee-saved" registers beyond epilogue.
; AMD64: jmpq
-; AMD64-NOT: %{{e[a-z]|rbx|rbp|r10|r12|r13|r14|r15}}
+; AMD64-NOT: %{{e[a-z]|rbx|rbp|r12|r13|r14|r15}}
; WIN64: jmpq
; WIN64-NOT: %{{e[a-z]|rbx|rsi|rdi|rbp|r12|r13|r14|r15}}
``````````
</details>
https://github.com/llvm/llvm-project/pull/208246
More information about the llvm-commits
mailing list