[llvm] [AArch64] Enable software pipelining for the Cortex-A320 (PR #206072)
William Huynh via llvm-commits
llvm-commits at lists.llvm.org
Fri Jun 26 07:23:18 PDT 2026
https://github.com/saturn691 updated https://github.com/llvm/llvm-project/pull/206072
>From 81b7f4252cbdbefc2a8a598407f85d4a5058d6ab Mon Sep 17 00:00:00 2001
From: William Huynh <William.Huynh at arm.com>
Date: Fri, 26 Jun 2026 14:55:42 +0100
Subject: [PATCH 1/2] [AArch64] Enable software pipelining for the Cortex-A320
Software pipelining gives mostly positive performance improvements
for the Cortex-A320. This enables MachinePipeliner for default for
the Cortex-A320 and keeps the behaviour of -aarch64-enable-pipeliner
for other CPUs.
---
llvm/lib/Target/AArch64/AArch64Features.td | 4 ++++
llvm/lib/Target/AArch64/AArch64Processors.td | 1 +
llvm/lib/Target/AArch64/AArch64Subtarget.cpp | 8 +++++++-
llvm/lib/Target/AArch64/AArch64TargetMachine.cpp | 5 -----
4 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64Features.td b/llvm/lib/Target/AArch64/AArch64Features.td
index c351a92f2f673..35247e6b8825f 100644
--- a/llvm/lib/Target/AArch64/AArch64Features.td
+++ b/llvm/lib/Target/AArch64/AArch64Features.td
@@ -839,6 +839,10 @@ def FeatureNoNegativeImmediates : SubtargetFeature<"no-neg-immediates",
"equivalent when the immediate does "
"not fit in the encoding.">;
+// Use the MachinePipeliner for instruction scheduling for the subtarget.
+def FeatureUseMIPipeliner: SubtargetFeature<"use-mipipeliner", "UseMIPipeliner", "true",
+ "Use the MachinePipeliner">;
+
// Address operands with shift amount 2 or 3 are fast on all Arm chips except
// some old Apple cores (A7-A10?) which handle all shifts slowly. Cortex-A57
// and derived designs through Cortex-X1 take an extra micro-op for shifts
diff --git a/llvm/lib/Target/AArch64/AArch64Processors.td b/llvm/lib/Target/AArch64/AArch64Processors.td
index dfa2bd46c8e56..e19bafb55aaff 100644
--- a/llvm/lib/Target/AArch64/AArch64Processors.td
+++ b/llvm/lib/Target/AArch64/AArch64Processors.td
@@ -22,6 +22,7 @@ def TuneA320 : SubtargetFeature<"a320", "ARMProcFamily", "CortexA320",
FeatureFuseAES,
FeatureFuseAdrpAdd,
FeaturePostRAScheduler,
+ FeatureUseMIPipeliner,
FeatureUseWzrToVecMove,
FeatureUseFixedOverScalableIfEqualCost,
FeatureAggressiveInterleaving]>;
diff --git a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
index f4b0af0333f4e..48760bd72c975 100644
--- a/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
+++ b/llvm/lib/Target/AArch64/AArch64Subtarget.cpp
@@ -97,6 +97,11 @@ static cl::opt<bool>
cl::init(false), cl::Hidden,
cl::desc("Enable subreg liveness tracking"));
+static cl::opt<bool>
+ EnableMachinePipeliner("aarch64-enable-pipeliner",
+ cl::desc("Enable Machine Pipeliner for AArch64"),
+ cl::init(false), cl::Hidden);
+
static cl::opt<bool>
UseScalarIncVL("sve-use-scalar-inc-vl", cl::init(false), cl::Hidden,
cl::desc("Prefer add+cnt over addvl/inc/dec"));
@@ -648,5 +653,6 @@ bool AArch64Subtarget::isX16X17Safer() const {
}
bool AArch64Subtarget::enableMachinePipeliner() const {
- return getSchedModel().hasInstrSchedModel();
+ return getSchedModel().hasInstrSchedModel() &&
+ (UseMIPipeliner || EnableMachinePipeliner);
}
diff --git a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
index 898d92e74c85c..0235f62bd6077 100644
--- a/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
+++ b/llvm/lib/Target/AArch64/AArch64TargetMachine.cpp
@@ -218,11 +218,6 @@ static cl::opt<bool>
cl::desc("Enable sinking and folding of instruction copies"),
cl::init(true), cl::Hidden);
-static cl::opt<bool>
- EnableMachinePipeliner("aarch64-enable-pipeliner",
- cl::desc("Enable Machine Pipeliner for AArch64"),
- cl::init(false), cl::Hidden);
-
static cl::opt<bool> EnableSRLTSubregToRegMitigation(
"aarch64-srlt-mitigate-sr2r",
cl::desc("Enable SUBREG_TO_REG mitigation by adding 'implicit-def' for "
>From e000ed4d7b66bdf2bf9fb8da6e5375620a08ba6d Mon Sep 17 00:00:00 2001
From: William Huynh <William.Huynh at arm.com>
Date: Fri, 26 Jun 2026 15:06:53 +0100
Subject: [PATCH 2/2] [AArch64] Allow high-II schedules only when vectorised
MachinePipeliner has a cap of II=27 which is reasonable, but it rejects
a subset of profitable loops. These are vectorised loops, that have not
been interleaved by the Loop Vectorizer (i.e. loops written with
intrinsics).
However, this is still a draft, as we need to uncap the II limit in code.
Also this is codex code, so there may be a better way to do this.
Assisted-by: codex, reviewed and tested by me
---
llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 55 ++++++++++++++++++++
1 file changed, 55 insertions(+)
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
index ca1a729dd1dd1..cea03d08a902a 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp
@@ -36,6 +36,7 @@
#include "llvm/CodeGen/MachineMemOperand.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachinePipeliner.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RegisterScavenging.h"
#include "llvm/CodeGen/StackMaps.h"
@@ -103,6 +104,11 @@ static cl::opt<unsigned> GatherOptSearchLimit(
cl::desc("Restrict range of instructions to search for the "
"machine-combiner gather pattern optimization"));
+static cl::opt<int> AArch64PipelinerMaxScalarII(
+ "aarch64-pipeliner-max-scalar-ii", cl::Hidden, cl::init(-1),
+ cl::desc("Reject scalar-only AArch64 MachinePipeliner schedules above this "
+ "initiation interval (-1 disables the check)"));
+
AArch64InstrInfo::AArch64InstrInfo(const AArch64Subtarget &STI)
: AArch64GenInstrInfo(STI, RI, AArch64::ADJCALLSTACKDOWN,
AArch64::ADJCALLSTACKUP, AArch64::CATCHRET),
@@ -11695,6 +11701,8 @@ class AArch64PipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
/// The normalized condition used by createTripCountGreaterCondition()
SmallVector<MachineOperand, 4> Cond;
+ bool containsFpOrVectorRegisters(SwingSchedulerDAG &SSD) const;
+
public:
AArch64PipelinerLoopInfo(MachineBasicBlock *LoopBB, MachineInstr *CondBranch,
MachineInstr *Comp, unsigned CompCounterOprNum,
@@ -11715,6 +11723,15 @@ class AArch64PipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
return MI == Comp;
}
+ bool shouldUseSchedule(SwingSchedulerDAG &SSD, SMSchedule &SMS) override {
+ if (AArch64PipelinerMaxScalarII >= 0 &&
+ SMS.getInitiationInterval() > AArch64PipelinerMaxScalarII &&
+ !containsFpOrVectorRegisters(SSD))
+ return false;
+
+ return true;
+ }
+
std::optional<bool> createTripCountGreaterCondition(
int TC, MachineBasicBlock &MBB,
SmallVectorImpl<MachineOperand> &CondParam) override {
@@ -11735,6 +11752,44 @@ class AArch64PipelinerLoopInfo : public TargetInstrInfo::PipelinerLoopInfo {
bool isMVEExpanderSupported() override { return true; }
};
+
+bool AArch64PipelinerLoopInfo::containsFpOrVectorRegisters(
+ SwingSchedulerDAG &SSD) const {
+ auto IsFpOrVectorRC = [](const TargetRegisterClass *RC) {
+ return RC && (AArch64::FPR8RegClass.hasSubClassEq(RC) ||
+ AArch64::FPR16RegClass.hasSubClassEq(RC) ||
+ AArch64::FPR32RegClass.hasSubClassEq(RC) ||
+ AArch64::FPR64RegClass.hasSubClassEq(RC) ||
+ AArch64::FPR128RegClass.hasSubClassEq(RC) ||
+ AArch64::PPRRegClass.hasSubClassEq(RC) ||
+ AArch64::PNRRegClass.hasSubClassEq(RC) ||
+ AArch64::ZPRRegClass.hasSubClassEq(RC) ||
+ AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
+ AArch64::ZPR3RegClass.hasSubClassEq(RC) ||
+ AArch64::ZPR4RegClass.hasSubClassEq(RC));
+ };
+
+ for (SUnit &SU : SSD.SUnits) {
+ const MachineInstr *MI = SU.getInstr();
+ for (const MachineOperand &MO : MI->operands()) {
+ if (!MO.isReg() || !MO.getReg())
+ continue;
+
+ Register Reg = MO.getReg();
+ const TargetRegisterClass *RC = nullptr;
+ if (Reg.isVirtual())
+ RC = MRI.getRegClassOrNull(Reg);
+ else if (Reg.isPhysical())
+ RC = TRI->getMinimalPhysRegClass(Reg.asMCReg());
+
+ if (IsFpOrVectorRC(RC))
+ return true;
+ }
+ }
+
+ return false;
+}
+
} // namespace
/// Clone an instruction from MI. The register of ReplaceOprNum-th operand
More information about the llvm-commits
mailing list