[llvm] [AArch64][MacroFusion] Fuse only tied AES pairs post-RA (PR #201610)
Tomer Shafir via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 06:50:05 PDT 2026
https://github.com/tomershafir updated https://github.com/llvm/llvm-project/pull/201610
>From 569b0fd8dd4e56ece35d32082ca709ca1b5df345 Mon Sep 17 00:00:00 2001
From: tomershafir <tomer.shafir8 at gmail.com>
Date: Thu, 4 Jun 2026 17:58:15 +0300
Subject: [PATCH 1/3] [AArch64][MacroFusion] Fuse only tied AES pairs post-RA
This patch adds an ad-hoc check to macro fusion to only fuse AES pairs that are tied post-RA as a guardrail.
Currently, ISel captures every RAW dependent AESE/D+AES[I]MC pair (by data-dependence DAG), and applies a constraint that the pair must write to the same dest, i.e the second instruction is tied (a thing that cannot be expressed in SSA IR). So this is effectively a NFC in that perspective, as AES is not really being lowered through other paths. Here we add an appropriate check to macro fusion, if registers are physical, to avoid pre-RA regression (maintaining the current status where pre-RA fusion hides theoretical better schedules even if the pari is not tied). Otherwise the tests in llvm/test/CodeGen/AArch64/misched-fusion-aes.ll may not catch an ISel change that would happen to pass, satisfying the register allocation being filechecked.
If it appears in the future that a subtarget can fuse untied pairs, we should re-address and maybe distinguish 2 subtarget features for the 2 cases.
Plus adding a test runline for latest apple-m5.
---
.../lib/Target/AArch64/AArch64MacroFusion.cpp | 24 +++++-
.../AArch64/misched-fusion-aes-post-ra.mir | 73 +++++++++++++++++++
.../CodeGen/AArch64/misched-fusion-aes.ll | 1 +
3 files changed, 96 insertions(+), 2 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/misched-fusion-aes-post-ra.mir
diff --git a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
index 55214f3f749e4..f8ad23cfe2561 100644
--- a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
@@ -117,6 +117,18 @@ static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
return false;
}
+// True unless the pair provably writes different physical registers. Pre-RA
+// the dests are still virtual, and post-RA it requires a genuine WAW (same dest
+// reg).
+static bool mayHaveWAWDependency(const MachineInstr &FirstMI,
+ const MachineInstr &SecondMI) {
+ Register DestFirst = FirstMI.getOperand(0).getReg();
+ Register DestSecond = SecondMI.getOperand(0).getReg();
+ if (!DestFirst.isPhysical() || !DestSecond.isPhysical())
+ return true;
+ return DestFirst == DestSecond;
+}
+
/// AES crypto encoding or decoding.
static bool isAESPair(const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
@@ -125,11 +137,19 @@ static bool isAESPair(const MachineInstr *FirstMI,
// AES encode.
case AArch64::AESMCrr:
case AArch64::AESMCrrTied:
- return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;
+ if (FirstMI == nullptr)
+ return true;
+ if (FirstMI->getOpcode() != AArch64::AESErr)
+ return false;
+ return mayHaveWAWDependency(*FirstMI, SecondMI);
// AES decode.
case AArch64::AESIMCrr:
case AArch64::AESIMCrrTied:
- return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;
+ if (FirstMI == nullptr)
+ return true;
+ if (FirstMI->getOpcode() != AArch64::AESDrr)
+ return false;
+ return mayHaveWAWDependency(*FirstMI, SecondMI);
}
return false;
diff --git a/llvm/test/CodeGen/AArch64/misched-fusion-aes-post-ra.mir b/llvm/test/CodeGen/AArch64/misched-fusion-aes-post-ra.mir
new file mode 100644
index 0000000000000..7b7e498474f3e
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/misched-fusion-aes-post-ra.mir
@@ -0,0 +1,73 @@
+# REQUIRES: asserts
+
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mattr=+fuse-aes,+crypto -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=generic -mattr=+crypto -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=cortex-a53 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=cortex-a57 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=cortex-a65 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=cortex-a72 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=cortex-a73 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=cortex-a76 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=cortex-a77 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=cortex-a78 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=cortex-a78c -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=cortex-x1 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=neoverse-e1 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=neoverse-n1 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=neoverse-v1 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=neoverse-512tvb -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=exynos-m3 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=exynos-m4 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=exynos-m5 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=ampere1 -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=ampere1a -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=ampere1b -misched-print-dags 2>&1 | FileCheck %s
+# RUN: llc -o /dev/null %s -run-pass=machine-scheduler -mtriple aarch64-unknown -mcpu=apple-m5 -misched-print-dags 2>&1 | FileCheck %s
+
+## Verify that only tied AES pairs (both instructions write to the same register)
+## are being fused post-RA.
+
+---
+name: encode_physregs_tied
+body: |
+ bb.0:
+ ; CHECK: SU(0): $q0 = AESErr undef $q0(tied-def 0), undef $q1
+ ; CHECK: Successors:
+ ; CHECK: SU(1): Ord Latency=0 Cluster
+ ; CHECK: SU(1): $q0 = AESMCrr $q0
+ $q0 = AESErr undef $q0, undef $q1
+ $q0 = AESMCrr $q0
+...
+---
+name: decode_physregs_tied
+body: |
+ bb.0:
+ ; CHECK: SU(0): $q0 = AESDrr undef $q0(tied-def 0), undef $q1
+ ; CHECK: Successors:
+ ; CHECK: SU(1): Ord Latency=0 Cluster
+ ; CHECK: SU(1): $q0 = AESIMCrr $q0
+ $q0 = AESDrr undef $q0, undef $q1
+ $q0 = AESIMCrr $q0
+...
+---
+name: encode_physregs_untied
+body: |
+ bb.0:
+ ; CHECK: SU(0): $q0 = AESErr undef $q0(tied-def 0), undef $q1
+ ; CHECK: Successors:
+ ; CHECK-NOT: SU({{.*}}): Ord Latency=0 Cluster
+ ; CHECK: SU(1): $q2 = AESMCrr $q0
+ $q0 = AESErr undef $q0, undef $q1
+ $q2 = AESMCrr $q0
+...
+---
+name: decode_physregs_untied
+body: |
+ bb.0:
+ ; CHECK: SU(0): $q0 = AESDrr undef $q0(tied-def 0), undef $q1
+ ; CHECK: Successors:
+ ; CHECK-NOT: SU({{.*}}): Ord Latency=0 Cluster
+ ; CHECK: SU(1): $q2 = AESIMCrr $q0
+ $q0 = AESDrr undef $q0, undef $q1
+ $q2 = AESIMCrr $q0
+...
\ No newline at end of file
diff --git a/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll b/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll
index dc6fa9128e933..cd1e46f187b09 100644
--- a/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll
+++ b/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll
@@ -20,6 +20,7 @@
; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=ampere1 | FileCheck %s
; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=ampere1a | FileCheck %s
; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=ampere1b | FileCheck %s
+; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=apple-m5 | FileCheck %s
declare <16 x i8> @llvm.aarch64.crypto.aese(<16 x i8> %d, <16 x i8> %k)
declare <16 x i8> @llvm.aarch64.crypto.aesmc(<16 x i8> %d)
>From 381152c62afbc6938d6fa82bed2cafff7f2face9 Mon Sep 17 00:00:00 2001
From: tomershafir <tomer.shafir8 at gmail.com>
Date: Mon, 8 Jun 2026 16:01:00 +0300
Subject: [PATCH 2/3] short-circuit pre-RA and restrict mayHave->have as
post-RA only
---
.../lib/Target/AArch64/AArch64MacroFusion.cpp | 21 ++++++++++---------
1 file changed, 11 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
index f8ad23cfe2561..82d3c6f21c6bc 100644
--- a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
@@ -117,15 +117,13 @@ static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
return false;
}
-// True unless the pair provably writes different physical registers. Pre-RA
-// the dests are still virtual, and post-RA it requires a genuine WAW (same dest
-// reg).
-static bool mayHaveWAWDependency(const MachineInstr &FirstMI,
- const MachineInstr &SecondMI) {
+// True if the pair writes to the same physical register.
+static bool haveWAWDependency(const MachineInstr &FirstMI,
+ const MachineInstr &SecondMI) {
Register DestFirst = FirstMI.getOperand(0).getReg();
Register DestSecond = SecondMI.getOperand(0).getReg();
- if (!DestFirst.isPhysical() || !DestSecond.isPhysical())
- return true;
+ assert(DestFirst.isPhysical() && DestSecond.isPhysical() &&
+ "Pair should have physical registers post-RA");
return DestFirst == DestSecond;
}
@@ -133,7 +131,8 @@ static bool mayHaveWAWDependency(const MachineInstr &FirstMI,
static bool isAESPair(const MachineInstr *FirstMI,
const MachineInstr &SecondMI) {
// Assume the 1st instr to be a wildcard if it is unspecified.
- switch (SecondMI.getOpcode()) {
+ unsigned SecondOpcode = SecondMI.getOpcode();
+ switch (SecondOpcode) {
// AES encode.
case AArch64::AESMCrr:
case AArch64::AESMCrrTied:
@@ -141,7 +140,8 @@ static bool isAESPair(const MachineInstr *FirstMI,
return true;
if (FirstMI->getOpcode() != AArch64::AESErr)
return false;
- return mayHaveWAWDependency(*FirstMI, SecondMI);
+ return SecondOpcode == AArch64::AESMCrrTied ||
+ haveWAWDependency(*FirstMI, SecondMI);
// AES decode.
case AArch64::AESIMCrr:
case AArch64::AESIMCrrTied:
@@ -149,7 +149,8 @@ static bool isAESPair(const MachineInstr *FirstMI,
return true;
if (FirstMI->getOpcode() != AArch64::AESDrr)
return false;
- return mayHaveWAWDependency(*FirstMI, SecondMI);
+ return SecondOpcode == AArch64::AESIMCrrTied ||
+ haveWAWDependency(*FirstMI, SecondMI);
}
return false;
>From 34b55dfd7d42665a9ad095ed29520d0cd09cfea3 Mon Sep 17 00:00:00 2001
From: tomershafir <tomer.shafir8 at gmail.com>
Date: Mon, 8 Jun 2026 16:49:48 +0300
Subject: [PATCH 3/3] revert assertion
---
llvm/lib/Target/AArch64/AArch64MacroFusion.cpp | 18 +++++++++---------
1 file changed, 9 insertions(+), 9 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
index 82d3c6f21c6bc..6b6923aceb9a0 100644
--- a/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
+++ b/llvm/lib/Target/AArch64/AArch64MacroFusion.cpp
@@ -117,13 +117,15 @@ static bool isArithmeticCbzPair(const MachineInstr *FirstMI,
return false;
}
-// True if the pair writes to the same physical register.
-static bool haveWAWDependency(const MachineInstr &FirstMI,
- const MachineInstr &SecondMI) {
+// True unless the pair provably writes different physical registers. Pre-RA
+// the dests are still virtual, and post-RA it requires a genuine WAW (same dest
+// reg).
+static bool mayHaveWAWDependency(const MachineInstr &FirstMI,
+ const MachineInstr &SecondMI) {
Register DestFirst = FirstMI.getOperand(0).getReg();
Register DestSecond = SecondMI.getOperand(0).getReg();
- assert(DestFirst.isPhysical() && DestSecond.isPhysical() &&
- "Pair should have physical registers post-RA");
+ if (!DestFirst.isPhysical() || !DestSecond.isPhysical())
+ return true;
return DestFirst == DestSecond;
}
@@ -140,8 +142,7 @@ static bool isAESPair(const MachineInstr *FirstMI,
return true;
if (FirstMI->getOpcode() != AArch64::AESErr)
return false;
- return SecondOpcode == AArch64::AESMCrrTied ||
- haveWAWDependency(*FirstMI, SecondMI);
+ return SecondOpcode == AArch64::AESMCrrTied || mayHaveWAWDependency(*FirstMI, SecondMI);
// AES decode.
case AArch64::AESIMCrr:
case AArch64::AESIMCrrTied:
@@ -149,8 +150,7 @@ static bool isAESPair(const MachineInstr *FirstMI,
return true;
if (FirstMI->getOpcode() != AArch64::AESDrr)
return false;
- return SecondOpcode == AArch64::AESIMCrrTied ||
- haveWAWDependency(*FirstMI, SecondMI);
+ return SecondOpcode == AArch64::AESIMCrrTied || mayHaveWAWDependency(*FirstMI, SecondMI);
}
return false;
More information about the llvm-commits
mailing list