[llvm] Adding slow indirect call tuning to zen5 (PR #195221)
via llvm-commits
llvm-commits at lists.llvm.org
Mon May 4 10:00:15 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: Anchu Rajendran S (anchuraj)
<details>
<summary>Changes</summary>
On znver5 processors, indirect calls from memory (CALL [mem]) are slower than loading the address into a register first and using a register indirect call (MOV + CALL *reg). This patch adds a new tuning feature TuningSlowIndirectCall that prevents folding loads into indirect call instructions with memory operand. This tuning is only needed for zen5 and hence is removed for zen6 or higher. Unlike TuningSlowTwoMemOps, TuningSlowIndirectCall only affects CALL instructions i and does not prevent PUSH from memory folding
---
Full diff: https://github.com/llvm/llvm-project/pull/195221.diff
9 Files Affected:
- (added) llvm/lib/Target/X86/.clang-format-ignore (+1)
- (modified) llvm/lib/Target/X86/X86.td (+13-2)
- (modified) llvm/lib/Target/X86/X86ISelDAGToDAG.cpp (+2-1)
- (modified) llvm/lib/Target/X86/X86InstrInfo.cpp (+12-6)
- (modified) llvm/lib/Target/X86/X86InstrPredicates.td (+2-1)
- (modified) llvm/lib/Target/X86/X86TargetTransformInfo.h (+1)
- (modified) llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload64.ll (+2)
- (modified) llvm/test/CodeGen/X86/fold-push.ll (+2)
- (added) llvm/test/CodeGen/X86/slow-indirect-call.ll (+87)
``````````diff
diff --git a/llvm/lib/Target/X86/.clang-format-ignore b/llvm/lib/Target/X86/.clang-format-ignore
new file mode 100644
index 0000000000000..9b18e68742ff9
--- /dev/null
+++ b/llvm/lib/Target/X86/.clang-format-ignore
@@ -0,0 +1 @@
+X86TargetTransformInfo.h
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index c42d3453e0729..50fb7204ebfa1 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -538,6 +538,14 @@ def TuningSlowTwoMemOps : SubtargetFeature<"slow-two-mem-ops",
"SlowTwoMemOps", "true",
"Two memory operand instructions are slow">;
+// On some processors, indirect calls from memory (CALL [mem]) are slow
+// compared to loading the address first and using a register indirect call.
+// This avoids folding loads into indirect call instructions, but does not
+// affect PUSH from memory folding.
+def TuningSlowIndirectCall : SubtargetFeature<"slow-indirect-call",
+ "SlowIndirectCall", "true",
+ "Indirect calls from memory are slow">;
+
// True if the LEA instruction inputs have to be ready at address generation
// (AG) time.
def TuningLEAUsesAG : SubtargetFeature<"lea-uses-ag", "LeaUsesAG", "true",
@@ -1643,7 +1651,9 @@ def ProcessorFeatures {
list<SubtargetFeature> ZN4Features =
!listconcat(ZN3Features, ZN4AdditionalFeatures);
- list<SubtargetFeature> ZN5Tuning = ZN4Tuning;
+ list<SubtargetFeature> ZN5AdditionalTuning = [TuningSlowIndirectCall];
+ list<SubtargetFeature> ZN5Tuning =
+ !listconcat(ZN4Tuning, ZN5AdditionalTuning);
list<SubtargetFeature> ZN5AdditionalFeatures = [FeatureMOVDIRI,
FeatureMOVDIR64B,
FeatureVP2INTERSECT,
@@ -1653,7 +1663,8 @@ def ProcessorFeatures {
list<SubtargetFeature> ZN5Features =
!listconcat(ZN4Features, ZN5AdditionalFeatures);
- list<SubtargetFeature> ZN6Tuning = ZN5Tuning;
+ list<SubtargetFeature> ZN6Tuning =
+ !listremove(ZN5Tuning, [TuningSlowIndirectCall]);
list<SubtargetFeature> ZN6AdditionalFeatures = [FeatureFP16,
FeatureAVXVNNIINT8,
FeatureAVXNECONVERT,
diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
index be95168f2de00..8955a6895d13a 100644
--- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp
@@ -1365,7 +1365,8 @@ void X86DAGToDAGISel::PreprocessISelDAG() {
// Only do this when the target can fold the load into the call or
// jmp.
!Subtarget->useIndirectThunkCalls() &&
- ((N->getOpcode() == X86ISD::CALL && !Subtarget->slowTwoMemOps()) ||
+ ((N->getOpcode() == X86ISD::CALL && !Subtarget->slowTwoMemOps() &&
+ !Subtarget->slowIndirectCall()) ||
(N->getOpcode() == X86ISD::TC_RETURN &&
(Subtarget->is64Bit() ||
!getTargetMachine().isPositionIndependent())))) {
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 5d4db845c5030..cd622e1c67dfd 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -7488,15 +7488,21 @@ MachineInstr *X86InstrInfo::foldMemoryOperandImpl(
unsigned Size, Align Alignment, bool AllowCommute, MachineInstr *&CopyMI,
VirtRegMap *VRM) const {
bool isSlowTwoMemOps = Subtarget.slowTwoMemOps();
+ bool isSlowIndirectCall = Subtarget.slowIndirectCall();
unsigned Opc = MI.getOpcode();
- // For CPUs that favor the register form of a call or push,
- // do not fold loads into calls or pushes, unless optimizing for size
- // aggressively.
- if (isSlowTwoMemOps && !MF.getFunction().hasMinSize() &&
+ // For CPUs that favor the register form of a call,
+ // do not fold loads into calls, unless optimizing for size aggressively.
+ if ((isSlowTwoMemOps || isSlowIndirectCall) &&
+ !MF.getFunction().hasMinSize() &&
(Opc == X86::CALL32r || Opc == X86::CALL64r ||
- Opc == X86::CALL64r_ImpCall || Opc == X86::PUSH16r ||
- Opc == X86::PUSH32r || Opc == X86::PUSH64r))
+ Opc == X86::CALL64r_ImpCall))
+ return nullptr;
+
+ // For CPUs that favor the register form of a push,
+ // do not fold loads into pushes, unless optimizing for size aggressively.
+ if (isSlowTwoMemOps && !MF.getFunction().hasMinSize() &&
+ (Opc == X86::PUSH16r || Opc == X86::PUSH32r || Opc == X86::PUSH64r))
return nullptr;
// Avoid partial and undef register update stalls unless optimizing for size.
diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td
index 2baf90eca15ea..352de4141ce71 100644
--- a/llvm/lib/Target/X86/X86InstrPredicates.td
+++ b/llvm/lib/Target/X86/X86InstrPredicates.td
@@ -252,7 +252,8 @@ let RecomputePerFunction = 1 in {
}
def CallImmAddr : Predicate<"Subtarget->isLegalToCallImmediateAddr()">;
-def FavorMemIndirectCall : Predicate<"!Subtarget->slowTwoMemOps()">;
+def FavorMemIndirectCall : Predicate<"!Subtarget->slowTwoMemOps() && "
+ "!Subtarget->slowIndirectCall()">;
def HasFastMem32 : Predicate<"!Subtarget->isUnalignedMem32Slow()">;
def HasFastLZCNT : Predicate<"Subtarget->hasFastLZCNT()">;
def HasFastSHLDRotate : Predicate<"Subtarget->hasFastSHLDRotate()">;
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.h b/llvm/lib/Target/X86/X86TargetTransformInfo.h
index ee1afdd8aaea9..d7c4bdde94a60 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.h
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.h
@@ -83,6 +83,7 @@ class X86TTIImpl final : public BasicTTIImplBase<X86TTIImpl> {
X86::TuningSlowPMULLD,
X86::TuningSlowSHLD,
X86::TuningSlowTwoMemOps,
+ X86::TuningSlowIndirectCall,
X86::TuningSlowUAMem16,
X86::TuningPreferMaskRegisters,
X86::TuningInsertVZEROUPPER,
diff --git a/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload64.ll b/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload64.ll
index 2379db0671c30..70c662ef9c82b 100644
--- a/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload64.ll
+++ b/llvm/test/CodeGen/X86/atom-call-reg-indirect-foldedreload64.ll
@@ -2,6 +2,8 @@
; RUN: FileCheck --check-prefix=ATOM %s
; RUN: llc < %s -mtriple=x86_64-linux-gnu -mcpu=core2 | \
; RUN: FileCheck --check-prefix=CORE2 %s
+; RUN: llc < %s -mtriple=x86_64-linux-gnu -mattr=slow-indirect-call | \
+; RUN: FileCheck --check-prefix=ATOM %s
; ATOM: callq *{{%[a-z]+[0-9]*}}
; CORE2: callq *funcp
;
diff --git a/llvm/test/CodeGen/X86/fold-push.ll b/llvm/test/CodeGen/X86/fold-push.ll
index c887b835aabd6..77526bf93ef90 100644
--- a/llvm/test/CodeGen/X86/fold-push.ll
+++ b/llvm/test/CodeGen/X86/fold-push.ll
@@ -1,5 +1,7 @@
; RUN: llc < %s -mtriple=i686-windows | FileCheck %s -check-prefix=CHECK -check-prefix=NORMAL
; RUN: llc < %s -mtriple=i686-windows -mattr=slow-two-mem-ops | FileCheck %s -check-prefix=CHECK -check-prefix=SLM
+; slow-indirect-call should not affect push folding (only slow-two-mem-ops does).
+; RUN: llc < %s -mtriple=i686-windows -mattr=slow-indirect-call | FileCheck %s -check-prefix=CHECK -check-prefix=NORMAL
declare void @foo(i32 %r)
diff --git a/llvm/test/CodeGen/X86/slow-indirect-call.ll b/llvm/test/CodeGen/X86/slow-indirect-call.ll
new file mode 100644
index 0000000000000..078ee35f4f358
--- /dev/null
+++ b/llvm/test/CodeGen/X86/slow-indirect-call.ll
@@ -0,0 +1,87 @@
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=+slow-indirect-call | FileCheck %s --check-prefix=SLOW
+; RUN: llc < %s -mtriple=x86_64-linux -mattr=-slow-indirect-call | FileCheck %s --check-prefix=NON-SLOW
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=znver5 | FileCheck %s --check-prefix=SLOW
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=znver4 | FileCheck %s --check-prefix=NON-SLOW
+; RUN: llc < %s -mtriple=i686-linux -mattr=+slow-indirect-call | FileCheck %s --check-prefix=SLOW32
+; RUN: llc < %s -mtriple=i686-linux -mattr=-slow-indirect-call | FileCheck %s --check-prefix=NON-SLOW32
+
+ at vtable = external dso_local global ptr
+
+; Indirect call through a global function pointer. With slow-indirect-call the
+; load must not be folded into the call.
+define i32 @test_call_global() nounwind {
+; SLOW-LABEL: test_call_global:
+; SLOW: movq vtable(%rip), %rax
+; SLOW: callq *%rax
+;
+; NON-SLOW-LABEL: test_call_global:
+; NON-SLOW: callq *vtable(%rip)
+;
+; SLOW32-LABEL: test_call_global:
+; SLOW32: movl vtable, %eax
+; SLOW32: calll *%eax
+;
+; NON-SLOW32-LABEL: test_call_global:
+; NON-SLOW32: calll *vtable
+entry:
+ %fp = load ptr, ptr @vtable, align 8
+ %ret = call i32 %fp(i32 42)
+ ret i32 %ret
+}
+
+; Vtable dispatch: load vtable pointer, load function from vtable, call.
+; The second load (vtable slot) must not be folded into the call.
+define i32 @test_call_vtable(ptr %obj) nounwind {
+; SLOW-LABEL: test_call_vtable:
+; SLOW: movq (%rdi), %rax
+; SLOW: movq (%rax), %rax
+; SLOW: callq *%rax
+;
+; NON-SLOW-LABEL: test_call_vtable:
+; NON-SLOW: movq (%rdi), %rax
+; NON-SLOW: callq *(%rax)
+;
+; SLOW32-LABEL: test_call_vtable:
+; SLOW32-NOT: calll *(%{{.*}})
+; SLOW32: calll *%{{.*}}
+;
+; NON-SLOW32-LABEL: test_call_vtable:
+; NON-SLOW32: calll *(%{{.*}})
+entry:
+ %vt = load ptr, ptr %obj, align 8
+ %fp = load ptr, ptr %vt, align 8
+ %ret = call i32 %fp(ptr %obj)
+ ret i32 %ret
+}
+
+; With minsize, the load should be folded even with slow-indirect-call.
+; The minsize attribute overrides the slow-indirect-call guard during
+; register allocation folding.
+define i32 @test_call_minsize() nounwind minsize {
+; SLOW-LABEL: test_call_minsize:
+; SLOW: callq *vtable(%rip)
+;
+; NON-SLOW-LABEL: test_call_minsize:
+; NON-SLOW: callq *vtable(%rip)
+entry:
+ %fp = load ptr, ptr @vtable, align 8
+ %ret = call i32 %fp(i32 42)
+ ret i32 %ret
+}
+
+; Tail calls should not be affected by slow-indirect-call. The load into the
+; jump target should still be folded.
+define void @test_tail_call(ptr %obj) nounwind {
+; SLOW-LABEL: test_tail_call:
+; SLOW: movq (%rdi), %rax
+; SLOW: jmpq *(%rax)
+;
+; NON-SLOW-LABEL: test_tail_call:
+; NON-SLOW: movq (%rdi), %rax
+; NON-SLOW: jmpq *(%rax)
+entry:
+ %vt = load ptr, ptr %obj, align 8
+ %fp = load ptr, ptr %vt, align 8
+ musttail call void %fp(ptr %obj)
+ ret void
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/195221
More information about the llvm-commits
mailing list